From 24c07f59f0134f33a05407c081ea03608b2e5c7d Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:09:37 -0500 Subject: [PATCH 01/18] fix(lock): stop hash collisions and same-node races from creating double holders The lock region has 256 records in the production layout, and tryAcquire treated a record held for a different resource as free. Any node could take over a held lock by touching a resource that hashed to the same record. Other double-holder windows fixed here: - two goroutines on one node acquiring the same key at once left the on-disk record cleared while both believed they held it - releasing a shared lock cleared the record for all readers when the releasing node was the last one to join - an exclusive request piggybacked on a same-node shared hold - reclaiming a stale shared record kept the dead readers' count - a release whose CAS lost a race returned nil without releasing Records now act as lock stripes: a held record blocks every resource that hashes to it, a node can hold several keys on one record, and a per-record gate serializes this node's on-disk acquire and release. Co-Authored-By: Claude Opus 5.5 --- internal/lock/lock.go | 314 +++++++++++++++++++--------------- internal/lock/safety_test.go | 315 +++++++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 81 +++++++++ 3 files changed, 575 insertions(+), 135 deletions(-) create mode 100644 internal/lock/safety_test.go diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 042fed5f..9405ab3c 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -206,6 +206,11 @@ type Manager struct { nodeSlot int // This node's slot heldLocks map[lockKey]*heldLock + // Several keys can hash to one record; the record stays held until the last key is released. + records map[uint64]*heldRecord + // gates serialize this node's on-disk operations per record. + gates map[uint64]*sync.Mutex + // Callback for cache invalidation onAcquire OnAcquireFunc @@ -233,6 +238,15 @@ type heldLock struct { goroutines map[uint64]int // Goroutine ID -> reference count for that goroutine } +type heldRecord struct { + mode LockMode + keys int +} + +func covers(held, want LockMode) bool { + return held == LockModeExclusive || held == want +} + // Errors var ( ErrLockTimeout = errors.New("lock acquisition timed out") @@ -251,6 +265,8 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(HintCacheSize), } } @@ -265,6 +281,8 @@ func NewManagerWithHintCacheSize(dev *storage.Device, offset, size uint64, nodeU nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(hintCacheSize), } } @@ -303,33 +321,25 @@ func (m *Manager) HintStats() (hits, misses uint64, hitRate float64) { // subsequent goroutines increment a local reference count. This allows // concurrent FUSE operations from the same node to share locks efficiently // while still providing cross-node serialization. +// +// Resources that hash to the same record share it: a node holding the record +// for one resource excludes other nodes from every resource in that record. func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uint64, mode LockMode) error { key := lockKey{Type: lockType, ResourceID: resourceID} - + recordID := m.hashToRecord(lockType, resourceID) gid := getGoroutineID() m.mu.Lock() - // Check if we already hold this lock (same node) - just increment refCount - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - reusing existing lock (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) + if m.reuseHeldKeyLocked(key, mode, gid) { m.mu.Unlock() return nil } - - // Check lock ordering to prevent deadlocks if err := m.checkLockOrder(lockType); err != nil { m.mu.Unlock() return err } m.mu.Unlock() - // Calculate record position (hash-based) - recordID := m.hashToRecord(lockType, resourceID) retryCount := 0 startTime := time.Now() @@ -338,63 +348,30 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin for { // Always attempt acquire first, then check context after failure // This ensures we try at least once even if context is nearly expired - acquired, err := m.tryAcquire(recordID, lockType, resourceID, mode) + held, added, err := m.attemptAcquire(key, recordID, mode, gid) if err != nil { - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - tryAcquire error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - attempt error: %v", lockType, resourceID, err) return err } - if acquired { - m.mu.Lock() - // Double-check in case another goroutine acquired while we were trying - if held, ok := m.heldLocks[key]; ok { - // Another goroutine beat us - just increment refCount - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - late arrival, reusing (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) - m.mu.Unlock() - // Release the on-disk lock we just acquired since we'll share the existing one - m.releaseRecord(recordID) - return nil - } - m.heldLocks[key] = &heldLock{ - Mode: mode, - RecordID: recordID, - refCount: 1, - goroutines: map[uint64]int{gid: 1}, - } - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired distributed lock (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) + if held { + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) // Trigger cache invalidation callback for exclusive locks. // We do this before returning so the caller reads fresh data. // Only exclusive locks trigger invalidation since shared locks // are for reading and can safely use cached values. + m.mu.Lock() onAcquire := m.onAcquire m.mu.Unlock() - - if onAcquire != nil && mode == LockModeExclusive { + if added && onAcquire != nil && mode == LockModeExclusive { onAcquire(lockType, resourceID, mode) } return nil } - // Check again if another goroutine acquired while we were waiting - m.mu.Lock() - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - piggyback on existing (refCount=%d)", lockType, resourceID, held.refCount) - m.mu.Unlock() - return nil - } - m.mu.Unlock() - retryCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - loop iteration, retryCount now %d", lockType, resourceID, retryCount) - // Check context cancellation, but allow at least 3 retries for same-node - // CAS races where the other goroutine is still adding to heldLocks + // Check context cancellation, but allow at least 3 retries if retryCount >= 3 { select { case <-ctx.Done(): @@ -416,6 +393,85 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin } } +// attemptAcquire makes one acquisition attempt. added reports whether this +// call registered the key rather than joining another goroutine's hold. +func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + if m.reuseHeldKeyLocked(key, mode, gid) { + m.mu.Unlock() + return true, false, nil + } + if rec, ok := m.records[recordID]; ok { + if !covers(rec.mode, mode) { + selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + m.mu.Unlock() + if selfDeadlock { + return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) + } + return false, false, nil + } + rec.keys++ + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil + } + m.mu.Unlock() + + acquired, err := m.tryAcquire(recordID, key.Type, key.ResourceID, mode) + if err != nil || !acquired { + return false, false, err + } + + m.mu.Lock() + m.records[recordID] = &heldRecord{mode: mode, keys: 1} + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil +} + +func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { + held, ok := m.heldLocks[key] + if !ok || !covers(m.records[held.RecordID].mode, mode) { + return false + } + held.refCount++ + held.goroutines[gid]++ + return true +} + +func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, gid uint64) { + m.heldLocks[key] = &heldLock{ + Mode: mode, + RecordID: recordID, + refCount: 1, + goroutines: map[uint64]int{gid: 1}, + } +} + +func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { + for _, held := range m.heldLocks { + if held.RecordID == recordID && held.goroutines[gid] > 0 { + return true + } + } + return false +} + +func (m *Manager) recordGate(recordID uint64) *sync.Mutex { + m.mu.Lock() + defer m.mu.Unlock() + gate, ok := m.gates[recordID] + if !ok { + gate = &sync.Mutex{} + m.gates[recordID] = gate + } + return gate +} + // Release releases a held lock. // With reference counting, this decrements the local count and only releases // the distributed lock when the last goroutine releases. @@ -431,36 +487,51 @@ func (m *Manager) Release(lockType LockType, resourceID uint64) error { return ErrNotLockOwner } - // Decrement goroutine-specific count - if held.goroutines != nil { - held.goroutines[gid]-- - if held.goroutines[gid] <= 0 { - delete(held.goroutines, gid) - } + held.goroutines[gid]-- + if held.goroutines[gid] <= 0 { + delete(held.goroutines, gid) } // Decrement overall reference count held.refCount-- if held.refCount > 0 { - // Other goroutines still using the lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - decremented refCount to %d", lockType, resourceID, gid, held.refCount) m.mu.Unlock() return nil } - // Last reference - release the distributed lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - last ref, releasing distributed lock", lockType, resourceID, gid) delete(m.heldLocks, key) recordID := held.RecordID m.mu.Unlock() - err := m.releaseRecord(recordID) + err := m.dropRecordKey(recordID) if err != nil { - log.Printf("[DEBUG] lock.Release: type=%d res=%d - releaseRecord error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Release: type=%d res=%d - release error: %v", lockType, resourceID, err) } return err } +func (m *Manager) dropRecordKey(recordID uint64) error { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + rec, ok := m.records[recordID] + if !ok { + m.mu.Unlock() + return ErrNotLockOwner + } + rec.keys-- + if rec.keys > 0 { + m.mu.Unlock() + return nil + } + delete(m.records, recordID) + m.mu.Unlock() + + return m.releaseRecord(recordID, rec.mode) +} + // tryAcquire attempts a single lock acquisition. // First tries an optimistic path using cached sequence hints, then falls back // to reading the current record if the hint is stale. @@ -499,40 +570,26 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint now := time.Now() - // Check if lock is free or can be acquired - canAcquire := false - reason := "unknown" + // A record held for a colliding resource is still held; taking it over lets two nodes hold it. + var sharedCount uint32 + reason := "" switch { case record.Mode == LockModeNone: - // Lock is free - canAcquire = true reason = "free" - case record.Type != lockType || record.ResourceID != resourceID: - // Hash collision - different resource, treat as free - // In production, we'd use a more sophisticated collision resolution - canAcquire = true - reason = "hash_collision" - case m.isLockStale(record): - // Lock is stale (owner died) - canAcquire = true + // Owner presumed dead; its shared holders are gone too. reason = "stale" case mode == LockModeShared && record.Mode == LockModeShared: - // Adding another shared holder - canAcquire = true reason = "shared_add" + sharedCount = record.SharedCount - case record.OwnerUUID == m.nodeUUID: - // Same node owns the lock - this can happen due to a race between - // Release clearing heldLocks and releaseRecord clearing on-disk. - // Safe to re-acquire since the previous holder from this node is gone. - canAcquire = true - reason = "same_node_reacquire" - } + case record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID: + // Leftover from a failed release or an earlier mount; this node holds no key on it. + reason = "leftover_from_this_node" - if !canAcquire { + default: log.Printf("[DEBUG] lock.tryAcquire: type=%d res=%d - cannot acquire, held by node=%x mode=%d", lockType, resourceID, record.OwnerUUID[:8], record.Mode) return false, nil } @@ -551,7 +608,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint Sequence: record.Sequence + 1, } if mode == LockModeShared { - newRecord.SharedCount = record.SharedCount + 1 + newRecord.SharedCount = sharedCount + 1 } // CAS the record @@ -603,58 +660,45 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } -// releaseRecord releases a lock record. +// releaseRecord releases this node's hold on a lock record. // After successful release, caches the new sequence number to enable // optimistic acquisition on the next lock attempt. -func (m *Manager) releaseRecord(recordID uint64) error { - record, err := m.readRecord(recordID) - if err != nil { - return err - } - - // Verify we own it - if record.OwnerUUID != m.nodeUUID { - // For shared locks, just decrement count - if record.Mode == LockModeShared && record.SharedCount > 1 { - newRecord := *record - newRecord.SharedCount-- - newRecord.Sequence++ - success, err := m.casRecord(recordID, record, &newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeShared, // Still shared (decremented) - UpdateAt: time.Now(), - }) - } - return nil +func (m *Manager) releaseRecord(recordID uint64, mode LockMode) error { + for { + record, err := m.readRecord(recordID) + if err != nil { + return err } - return ErrNotLockOwner - } - // Clear the record - newRecord := &Record{ - Sequence: record.Sequence + 1, - } + var newRecord *Record + switch { + case mode == LockModeExclusive && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID, + mode == LockModeShared && record.Mode == LockModeShared && record.SharedCount <= 1: + newRecord = &Record{Sequence: record.Sequence + 1} + case mode == LockModeShared && record.Mode == LockModeShared: + // OwnerUUID names only the latest reader, so readers release by count. + next := *record + next.SharedCount-- + next.Sequence++ + newRecord = &next + default: + // Another node reclaimed the record after it went stale. + return ErrNotLockOwner + } - success, err := m.casRecord(recordID, record, newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - // Mode is None since lock is now free - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeNone, - UpdateAt: time.Now(), - }) + success, err := m.casRecord(recordID, record, newRecord) + if err != nil { + return err + } + if success { + m.hintCache.Put(recordID, &lockHint{ + Sequence: newRecord.Sequence, + Mode: newRecord.Mode, + UpdateAt: time.Now(), + }) + return nil + } } - return nil } // isLockStale checks if a lock's owner is dead. diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go new file mode 100644 index 00000000..29267588 --- /dev/null +++ b/internal/lock/safety_test.go @@ -0,0 +1,315 @@ +package lock + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/storage" +) + +func newTestNode(dev *storage.Device, offset, size uint64, n int) *Manager { + var uuid [16]byte + copy(uuid[:], fmt.Sprintf("test-node-%06d", n)) + return NewManager(dev, offset, size, uuid, n) +} + +func acquireWithin(m *Manager, d time.Duration, lt LockType, id uint64, mode LockMode) error { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + return m.Acquire(ctx, lt, id, mode) +} + +func requireBlocked(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + err := acquireWithin(m, 50*time.Millisecond, lt, id, mode) + if err == nil { + m.Release(lt, id) + t.Fatalf("node %d acquired type=%d id=%d mode=%d while another holder should exclude it", m.nodeSlot, lt, id, mode) + } + if !errors.Is(err, ErrLockTimeout) { + t.Fatalf("Acquire: got %v, want ErrLockTimeout", err) + } +} + +func requireAcquired(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + if err := acquireWithin(m, time.Second, lt, id, mode); err != nil { + t.Fatalf("node %d Acquire type=%d id=%d mode=%d: %v", m.nodeSlot, lt, id, mode, err) + } +} + +// ageRecord rewrites the lock's record as if it was taken age ago and never renewed. +func ageRecord(t *testing.T, m *Manager, lt LockType, id uint64, age time.Duration) { + t.Helper() + recordID := m.hashToRecord(lt, id) + rec, err := m.readRecord(recordID) + if err != nil { + t.Fatalf("readRecord: %v", err) + } + rec.AcquireTime = time.Now().Add(-age).Unix() + buf := storage.GetBuffer(RecordSize) + defer storage.PutBuffer(buf) + copy(buf.Bytes(), serializeRecord(rec)) + if _, err := m.dev.WriteAt(buf.Bytes(), int64(m.offset)+int64(recordID)*RecordSize); err != nil { + t.Fatalf("write record: %v", err) + } +} + +const pastStale = StaleThreshold + 2*time.Second + +func TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + requireBlocked(t, b, LockTypeInode, 2, LockModeExclusive) +} + +func TestNodeHoldsCollidingResourcesUntilLastRelease(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + requireAcquired(t, a, LockTypeInode, 7, LockModeExclusive) + if err := a.Release(LockTypeInode, 7); err != nil { + t.Fatalf("Release inode: %v", err) + } + requireBlocked(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + + if err := a.Release(LockTypeBlockBitmap, 0); err != nil { + t.Fatalf("Release bitmap: %v", err) + } + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) +} + +func TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + const workers = 4 + for iter := 0; iter < 30; iter++ { + start := make(chan struct{}) + release := make(chan struct{}) + var acquired, done sync.WaitGroup + errs := make(chan error, workers) + acquired.Add(workers) + done.Add(workers) + for w := 0; w < workers; w++ { + go func() { + defer done.Done() + <-start + err := acquireWithin(a, time.Second, LockTypeInode, 42, LockModeExclusive) + acquired.Done() + if err != nil { + errs <- err + return + } + <-release + if err := a.Release(LockTypeInode, 42); err != nil { + errs <- err + } + }() + } + close(start) + acquired.Wait() + requireBlocked(t, b, LockTypeInode, 42, LockModeExclusive) + close(release) + done.Wait() + close(errs) + for err := range errs { + t.Fatalf("iteration %d: %v", iter, err) + } + } +} + +func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 5, LockModeShared) + requireAcquired(t, b, LockTypeInode, 5, LockModeShared) + if err := b.Release(LockTypeInode, 5); err != nil { + t.Fatalf("b Release: %v", err) + } + requireBlocked(t, c, LockTypeInode, 5, LockModeExclusive) + + if err := a.Release(LockTypeInode, 5); err != nil { + t.Fatalf("a Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) +} + +func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + + writer := func() error { + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + return <-errc + } + + if err := writer(); !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) + } + if err := a.Release(LockTypeInode, 9); err != nil { + t.Fatalf("Release shared: %v", err) + } + if err := writer(); err != nil { + t.Fatalf("exclusive Acquire after readers left: %v", err) + } +} + +func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) + if !errors.Is(err, ErrDeadlockRisk) { + t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + } +} + +func TestStaleLockIsReclaimedFromPausedHolder(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + ageRecord(t, a, LockTypeBlockBitmap, 0, pastStale) + + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + if held := a.HeldLocks(); len(held) != 1 { + t.Fatalf("paused holder's view: got %v, want it to still list the lock (no fencing)", held) + } +} + +func TestPausedHolderReleaseKeepsReclaimedLock(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 3, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 3, pastStale) + requireAcquired(t, b, LockTypeInode, 3, LockModeExclusive) + + if err := a.Release(LockTypeInode, 3); !errors.Is(err, ErrNotLockOwner) { + t.Fatalf("paused holder Release: got %v, want ErrNotLockOwner", err) + } + requireBlocked(t, c, LockTypeInode, 3, LockModeExclusive) +} + +func TestStaleSharedLockReclaimResetsReaderCount(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + dead := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, dead, LockTypeInode, 4, LockModeShared) + ageRecord(t, dead, LockTypeInode, 4, pastStale) + requireAcquired(t, a, LockTypeInode, 4, LockModeShared) + if err := a.Release(LockTypeInode, 4); err != nil { + t.Fatalf("Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 4, LockModeExclusive) +} + +func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { + const ( + guard = 4 * RecordSize + regionSize = 16 * RecordSize + resources = 1000 + cycles = 1500 + ) + dev, cleanup := createTestDevice(t, guard+regionSize+guard) + defer cleanup() + + sentinel := storage.NewAlignedBuffer(guard) + for i := range sentinel.Bytes() { + sentinel.Bytes()[i] = 0xA5 + } + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.WriteAt(sentinel.Bytes(), off); err != nil { + t.Fatalf("write sentinel: %v", err) + } + } + + nodes := []*Manager{ + newTestNode(dev, guard, regionSize, 1), + newTestNode(dev, guard, regionSize, 2), + } + var wg sync.WaitGroup + errs := make(chan error, len(nodes)) + for n, m := range nodes { + wg.Add(1) + go func(n int, m *Manager) { + defer wg.Done() + for i := 0; i < cycles; i++ { + id := uint64((i*7 + n*13) % resources) + lt := LockTypeInode + if i%3 == 0 { + lt = LockTypeExtent + } + if err := acquireWithin(m, 5*time.Second, lt, id, LockModeExclusive); err != nil { + errs <- fmt.Errorf("node %d cycle %d: %w", n, i, err) + return + } + m.Release(lt, id) + } + }(n, m) + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + check := newTestNode(dev, guard, regionSize, 3) + for id := uint64(0); id < resources; id++ { + for _, lt := range []LockType{LockTypeInode, LockTypeExtent} { + requireAcquired(t, check, lt, id, LockModeExclusive) + if err := check.Release(lt, id); err != nil { + t.Fatalf("Release type=%d id=%d: %v", lt, id, err) + } + } + } + + got := storage.NewAlignedBuffer(guard) + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.ReadAt(got.Bytes(), off); err != nil { + t.Fatalf("read sentinel: %v", err) + } + if !bytes.Equal(got.Bytes(), sentinel.Bytes()) { + t.Fatalf("lock manager wrote outside its region near offset %d", off) + } + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index a6ade5ac..0b76fdbb 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1307,3 +1307,84 @@ scenarios: priority: go_only environments: go: internal/journal/journal_fuzz_test.go::FuzzJournalScan + + # =========================================================================== + # DISTRIBUTED LOCK SAFETY (internal/lock) + # Two or three lock managers share one device to act as separate nodes. + # =========================================================================== + + # Docker/VM TODO: a shell test would have to freeze a node (docker pause, or + # SIGSTOP on the QEMU process) inside a lock-held section that lasts + # milliseconds, then resume it after StaleThreshold. Shell tests cannot target + # that window, so the Go test simulates the pause by aging the on-disk record. + - id: lock-stale-reclaim-paused-holder + name: Stale lock reclaimed from paused holder + description: A holder stalled past StaleThreshold loses its lock to another node without being told + priority: high + environments: + go: ../../internal/lock/safety_test.go::TestStaleLockIsReclaimedFromPausedHolder + docker: TODO + vm: TODO + + - id: lock-paused-holder-release + name: Paused holder release keeps new holder's lock + description: A late Release from a holder whose lock was reclaimed returns ErrNotLockOwner and leaves the new holder's record intact + priority: go_only + environments: + go: internal/lock/safety_test.go::TestPausedHolderReleaseKeepsReclaimedLock + + - id: lock-colliding-resources-excluded + name: Colliding resources exclude other nodes + description: A held record blocks other nodes from every resource that hashes to it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord + + - id: lock-same-node-colliding-resources + name: One node holds colliding resources + description: A node can hold two resources that share a record; the record stays held until the last one is released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNodeHoldsCollidingResourcesUntilLastRelease + + - id: lock-same-node-concurrent-acquire + name: Concurrent same-node acquire keeps the lock + description: Goroutines on one node acquiring the same lock at once leave it held on disk + priority: go_only + environments: + go: internal/lock/safety_test.go::TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut + + - id: lock-shared-release-by-count + name: Shared lock survives another reader's release + description: Releasing one reader's shared hold does not free the lock for writers while other readers hold it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases + + - id: lock-exclusive-waits-same-node-readers + name: Exclusive waits for same-node readers + description: An exclusive request does not piggyback on a shared hold from the same node + priority: go_only + environments: + go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + + - id: lock-upgrade-same-goroutine + name: Shared to exclusive upgrade rejected + description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + + - id: lock-stale-shared-reclaim + name: Stale shared lock reclaim drops dead readers + description: Reclaiming a stale shared record resets the reader count so it can be fully released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestStaleSharedLockReclaimResetsReaderCount + + - id: lock-region-bounded + name: Lock region stays bounded + description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestLockRegionStaysBoundedAcrossCycles From 78daa431ef5104d29df93f57747bce5255d91532 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:09:37 -0500 Subject: [PATCH 02/18] docs(operations): document lock stall hazard and safe operating limits Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 591915b3..f006c084 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -229,12 +229,42 @@ rm /mnt/fs/.snapshots/snapshot_name iscsiadm -m node -T -p --logout ``` +### Distributed Lock Limits + +Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, SFB containers, snapshots) through lock records on the shared device. Read this before you run more than one node. + +**How a lock goes stale.** A lock record stores the wall-clock time it was taken. Nothing renews it. Any other node may take over a lock that is older than 15 seconds (`StaleThreshold`), judged by that node's own clock. This is how locks held by a crashed node get freed. + +**The hazard: a stalled node keeps writing.** If a node stops for more than 15 seconds while it holds a lock, another node takes the lock. When the stalled node resumes, it finishes its writes. No write path checks that the lock is still held, and there is no fencing token. The node only learns it lost the lock when it releases it (`ErrNotLockOwner`), and callers ignore that error. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. + +Common causes of a stall: + +- VM pause, suspend, live migration, or a memory snapshot +- `docker pause`, cgroup freezer, `SIGSTOP`, or a debugger on the mount process +- Heavy swapping or a long GC pause +- Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover + +**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. + +**Safe operating limits:** + +| Limit | Why | +|-------|-----| +| Keep node clocks within 1 second (chrony or NTP) | Clock skew S shortens the stale threshold to 15s − S | +| No node may stall for 10 seconds or more while mounted | Leaves margin below the 15s threshold | +| Storage I/O must fail, not queue, within 10 seconds | A queued write that lands after takeover corrupts metadata. Do not use multipath `queue_if_no_path`; lower iSCSI `node.session.timeo.replacement_timeout` (default 120s) | +| Unmount before you pause, migrate or snapshot a node VM | Unmounting releases its locks | + +If a node did stall for more than 15 seconds while mounted (hung-task warnings in `dmesg`, a VM pause, a path failover), stop its mount process before it does more writes, then check the data from a healthy node. + +What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks 15 seconds after it acquired them. + ### Handling Node Failure When a node fails unexpectedly: 1. **Other nodes continue operating** - Distributed locking handles failed node -2. **Lock cleanup** - Stale locks are detected via heartbeat timeout +2. **Lock cleanup** - Other nodes take over the failed node's locks once they are 15 seconds old (lock age, not node heartbeat) 3. **No immediate action required** - System is self-healing **When failed node recovers**: From 8fc8444b0a636d820faeefeacd9fd3da3759fd86 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:21:50 -0500 Subject: [PATCH 03/18] fix(fs): wire the lock manager into mounted filesystems cmd/mount built the FS with NewWithComponents, which had no lock manager parameter, so every inode, extent and SFB lock branch in internal/goclusterfs was dead in production. Two nodes writing a new file's first block at the same time lost one node's data in 11 of 20 files. The dead branches had never run and lost data once enabled: after allocating a block without the inode lock, writeInternal and Allocate re-read the inode under the lock and dropped the new extent. They now map the new block in the fresh inode, or free it and retry if another node mapped that block first. Changes made under the lock are written before the lock is dropped for the next allocation, and the final size update builds on the inode read under the lock. Readers joining a shared lock no longer refresh its age. A reader count leaked by a lost CAS update now goes stale after 15s instead of blocking writers for as long as readers keep arriving; Docker hung on rmdir without this. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 2 +- docs/OPERATIONS.md | 2 +- internal/goclusterfs/fs.go | 123 +++++++++++++++++++++------- internal/lock/lock.go | 5 +- internal/lock/safety_test.go | 27 ++++++ tests/docker/scripts/run-tests.sh | 38 +++++++++ tests/integration/multinode_test.go | 74 +++++++++++++++++ tests/test-manifest.yaml | 16 ++++ tests/vm/scripts/run-vm-tests.sh | 38 +++++++++ 9 files changed, 292 insertions(+), 33 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 7d13bd36..42e7a9db 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -279,7 +279,7 @@ func main() { } // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency - gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, nodeUUID, dataOffset) + gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) // Set up cross-node recovery handler for failed node journal replay recoveryCtx := &journal.RecoveryContext{ diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index f006c084..095e12a0 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -244,7 +244,7 @@ Common causes of a stall: - Heavy swapping or a long GC pause - Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover -**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. +**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. **Safe operating limits:** diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 9b45c5f4..fec5f984 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -193,11 +193,13 @@ func NewWithComponents( tm *trim.Manager, // May be nil if TRIM is disabled sfbMgr *sfb.Manager, // May be nil if SFB is disabled refcountReg *refcount.Region, // May be nil if snapshots not enabled + lockMgr *lock.Manager, // May be nil for single-node use nodeUUID [16]byte, dataOffset uint64, ) *FS { return &FS{ dev: dev, + lockMgr: lockMgr, bitmapMgr: bm, inodeTbl: inodeTbl, extentMgr: extentMgr, @@ -2070,6 +2072,52 @@ func retryOnTreeConflict(err error) error { return err } +// allocateBlockLocked maps a new data block at blockNum and returns the inode +// re-read under the inode lock, which stays held on success. The bitmap comes +// before the inode in lock order, so the data block and any extent-tree blocks +// are allocated before the lock is taken. +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, error) { + blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) + if err != nil { + return nil, nil, syscall.ENOSPC + } + ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} + treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) + if err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, syscall.ENOSPC + } + defer releaseTree() + + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, syscall.EIO + } + fail := func(errno error) (*inode.Inode, *extent.Extent, error) { + f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, errno + } + fresh, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + return fail(syscall.EIO) + } + // Insert fails if another node mapped this block, or grew the tree past the reservation, meanwhile. + if err := f.fs.extentMgr.Insert(treeCtx, fresh, ext); err != nil { + f.fs.extentMgr.DiscardPending(ctx, fresh) + switch { + case errors.Is(err, extent.ErrExtentTreeFull): + return fail(syscall.EFBIG) + case errors.Is(err, extent.ErrCorruptTree): + return fail(syscall.EIO) + } + return fail(errRetryWrite) + } + fresh.AllocatedBlocks += ext.Length + fresh.WrittenBlocks += ext.Length + return fresh, ext, nil +} + // writeInternal performs the actual write operation. // Returns errRetryWrite if a concurrent modification is detected and the caller should retry. func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (int, error) { @@ -2077,6 +2125,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Track if we hold the inode lock lockHeld := false + // ino has changes made under the inode lock that are not on disk yet + inoDirty := false defer func() { if lockHeld && f.fs.lockMgr != nil { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) @@ -2249,42 +2299,46 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // bitmap (6) must be acquired before inode (8) // AllocateBlocks acquires bitmap lock internally if lockHeld && f.fs.lockMgr != nil { + // The inode is re-read after allocation, so persist changes made under the lock first. + if inoDirty { + if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { + log.Printf("[DEBUG] Write inode %d: persisting inode before allocation failed: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, retryOnTreeConflict(err) + } + inoDirty = false + } f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) lockHeld = false } - // Allocate block (acquires bitmap lock internally, then releases it) - ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) - if err != nil { - log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) - switch { - case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): - // Concurrent allocation by another node - return 0, errRetryWrite - case errors.Is(err, extent.ErrExtentTreeFull): - return 0, syscall.EFBIG - case errors.Is(err, extent.ErrCorruptTree): - return 0, syscall.EIO - default: - return 0, syscall.ENOSPC - } - } - - // Re-acquire inode lock after allocation if f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to re-acquire lock after allocation: %v", f.inodeNum, err) - return 0, syscall.EIO + freshIno, newExt, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + if allocErr != nil { + log.Printf("[DEBUG] Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) + return 0, allocErr } lockHeld = true - - // Re-read inode after re-acquiring lock to verify state - freshIno, readErr := f.fs.inodeTbl.Read(f.inodeNum) - if readErr != nil { - log.Printf("[DEBUG] Write inode %d: re-read after allocation failed: %v", f.inodeNum, readErr) - return 0, syscall.EIO - } ino = freshIno + inoDirty = true + ext = newExt + } else { + // Allocate block (acquires bitmap lock internally, then releases it) + ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) + if err != nil { + log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) + switch { + case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): + // Concurrent allocation by another node + return 0, errRetryWrite + case errors.Is(err, extent.ErrExtentTreeFull): + return 0, syscall.EFBIG + case errors.Is(err, extent.ErrCorruptTree): + return 0, syscall.EIO + default: + return 0, syscall.ENOSPC + } + } } extOffset = 0 @@ -2359,6 +2413,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } if cowResult.WasShared { + inoDirty = true log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) @@ -2417,6 +2472,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return 0, syscall.EIO } if cleared { + inoDirty = true log.Printf("[DEBUG] Write inode %d: cleared Unwritten flag for block %d", f.inodeNum, blockNum) } @@ -2518,6 +2574,10 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } return 0, syscall.EIO } + // ino was read before the lock, so it can miss another node's extents. + if acquiredLockForSize { + ino = freshIno + } // Use the freshest size from disk, then calculate new size as max(fresh_size, our_write_end) newSize := freshIno.Size if offset > newSize { @@ -3172,12 +3232,15 @@ func (f *File) allocate(ctx context.Context, off uint64, size uint64, mode uint3 } defer f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - // Re-read inode after lock to get latest state - ino, err = f.fs.inodeTbl.Read(f.inodeNum) + // ino holds the new extents, so keep it only if no node wrote the inode since it was read. + fresh, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Allocate inode %d: re-read failed: %v", f.inodeNum, err) return syscall.EIO, nil } + if fresh.MetaGeneration != ino.MetaGeneration { + return 0, extent.ErrStaleTree + } } // Update file size if not KEEP_SIZE and new size is larger diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 9405ab3c..270bd483 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -572,6 +572,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint // A record held for a colliding resource is still held; taking it over lets two nodes hold it. var sharedCount uint32 + acquireTime := now.Unix() reason := "" switch { case record.Mode == LockModeNone: @@ -584,6 +585,8 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint case mode == LockModeShared && record.Mode == LockModeShared: reason = "shared_add" sharedCount = record.SharedCount + // Keep the first reader's time: a count leaked by a lost CAS update must still go stale. + acquireTime = record.AcquireTime case record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID: // Leftover from a failed release or an earlier mount; this node holds no key on it. @@ -604,7 +607,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint ResourceID: resourceID, OwnerUUID: m.nodeUUID, OwnerSlot: uint32(m.nodeSlot), - AcquireTime: now.Unix(), + AcquireTime: acquireTime, Sequence: record.Sequence + 1, } if mode == LockModeShared { diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 29267588..7c32e451 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -313,3 +313,30 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { } } } + +func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + leaked := newTestNode(dev, 0, 1<<20, 1) + reader := newTestNode(dev, 0, 1<<20, 2) + writer := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, leaked, LockTypeInode, 2, LockModeShared) + recordID := leaked.hashToRecord(LockTypeInode, 2) + rec, err := leaked.readRecord(recordID) + if err != nil { + t.Fatalf("readRecord: %v", err) + } + rec.AcquireTime = time.Now().Add(-StaleThreshold + 2*time.Second).Unix() + if err := leaked.writeRecord(recordID, rec); err != nil { + t.Fatalf("writeRecord: %v", err) + } + + requireAcquired(t, reader, LockTypeInode, 2, LockModeShared) + if err := reader.Release(LockTypeInode, 2); err != nil { + t.Fatalf("reader Release: %v", err) + } + time.Sleep(3 * time.Second) + + requireAcquired(t, writer, LockTypeInode, 2, LockModeExclusive) +} diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 0f7406fc..4acb3212 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -1852,6 +1852,43 @@ test_concurrent_writes() { return 0 } +# Two nodes write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other node's extent. +test_concurrent_first_write_same_block() { + log "Testing concurrent first writes to the same block from two nodes..." + local writer0="${NODE_ARRAY[0]}" + local writer1="${NODE_ARRAY[1]}" + local reader="${NODE_ARRAY[2]}" + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + exec_on_node "$writer0" touch "/mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at "$writer0" "first_write_$i.bin" A "$size" "$offset0" & + write_pattern_at "$writer1" "first_write_$i.bin" B "$size" "$offset1" & + wait + done + sleep 2 + + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at "$reader" "first_write_$i.bin" A "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at "$reader" "first_write_$i.bin" B "$offset1" 4096 || lost=$((lost + 1)) + done + if [ "$lost" -gt 0 ]; then + log "ERROR: $lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log "All concurrent first writes survived" + return 0 +} + # Test 2: Container kill and recovery test_container_kill_recovery() { log "Testing container kill and recovery..." @@ -2717,6 +2754,7 @@ else run_test "Concurrent File Creation" test_concurrent_file_creation run_test "Concurrent Directory Ops" test_concurrent_directory_ops fi +run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Container Kill Recovery" test_container_kill_recovery run_test "Journal Recovery Multi-Node" test_journal_recovery_multinode run_test "Journal Recovery Multiple Files" test_journal_recovery_multiple_files diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index f421b0c4..c6b23f27 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -1531,3 +1531,77 @@ func TestStressRandomFailures(t *testing.T) { t.Logf("Stress test completed: %d files created", filesCreated) } + +// TestMultiNodeConcurrentFirstWriteToSameBlock has two nodes write different +// ranges of a new file's first block at the same time. Each node must take the +// inode lock around its read-modify-write of the extents, or the later inode +// write drops the other node's extent and its data. +func TestMultiNodeConcurrentFirstWriteToSameBlock(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 100) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + if err := cluster.WaitAllMounted(15 * time.Second); err != nil { + t.Fatalf("Nodes failed to mount: %v", err) + } + + const files = 20 + name := func(i int) string { return fmt.Sprintf("first_write_%02d.bin", i) } + for i := 0; i < files; i++ { + if err := cluster.Node(0).WriteFile(name(i), nil); err != nil { + t.Fatalf("Failed to create %s: %v", name(i), err) + } + } + // Outlast the kernel's 1s attribute cache so node 1 sees the files. + time.Sleep(1500 * time.Millisecond) + + // Offsets past 64KB keep the files off the SFB path. + const offset0, offset1 = 256 * 1024, 512 * 1024 + data0 := bytes.Repeat([]byte("N0"), 32*1024) + data1 := bytes.Repeat([]byte("N1"), 32*1024) + + for i := 0; i < files; i++ { + errs := cluster.RunConcurrent([]int{0, 1}, func(idx int, node *MountedFS) error { + if idx == 0 { + return node.WriteFileAt(name(i), data0, offset0) + } + return node.WriteFileAt(name(i), data1, offset1) + }) + for idx, err := range errs { + if err != nil { + t.Fatalf("%s: node %d write failed: %v", name(i), idx, err) + } + } + } + + // Read from a freshly mounted node so no cached extents hide the result. + cluster.StopAllNodes() + if err := cluster.StartNode(0, false); err != nil { + t.Fatalf("Failed to remount node 0: %v", err) + } + + lost := 0 + for i := 0; i < files; i++ { + got, err := cluster.Node(0).ReadFile(name(i)) + if err != nil { + t.Fatalf("Failed to read %s: %v", name(i), err) + } + if len(got) < offset1+len(data1) { + t.Errorf("%s: size %d, want at least %d", name(i), len(got), offset1+len(data1)) + lost++ + continue + } + ok0 := bytes.Equal(got[offset0:offset0+len(data0)], data0) + ok1 := bytes.Equal(got[offset1:offset1+len(data1)], data1) + if !ok0 || !ok1 { + t.Errorf("%s: node 0 data intact=%v, node 1 data intact=%v", name(i), ok0, ok1) + lost++ + } + } + if lost > 0 { + t.Fatalf("%d of %d files lost a node's write", lost, files) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 0b76fdbb..b5da29c3 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -304,6 +304,15 @@ scenarios: docker: run-tests.sh::test_mixed_size_writes vm: run-vm-tests.sh::test_mixed_size_writes + - id: multinode-concurrent-first-write-same-block + name: Concurrent first write to the same block + description: Two nodes write different ranges of a new file's first block at once; both writes survive + priority: high + environments: + go: multinode_test.go::TestMultiNodeConcurrentFirstWriteToSameBlock + docker: run-tests.sh::test_concurrent_first_write_same_block + vm: run-vm-tests.sh::test_concurrent_first_write_same_block + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets @@ -1382,6 +1391,13 @@ scenarios: environments: go: internal/lock/safety_test.go::TestStaleSharedLockReclaimResetsReaderCount + - id: lock-leaked-shared-count-goes-stale + name: Leaked reader count goes stale + description: New readers do not refresh a shared record's age, so a reader count leaked by a lost update cannot block writers forever + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNewReadersDoNotKeepLeakedSharedLockFresh + - id: lock-region-bounded name: Lock region stays bounded description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index aafbc100..d9018289 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2330,6 +2330,43 @@ test_stability() { # ============================================================================= # Test 10: Two VMs write to same file at different offsets +# Two VMs write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other VM's extent. +test_concurrent_first_write_same_block() { + log_info "Testing concurrent first writes to the same block (2 VMs)..." + ensure_cluster_healthy 2 + + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + ssh_run 1 "touch /mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$size" "$offset0" & + write_pattern_at_on_vm 2 "first_write_$i.bin" "B" "$size" "$offset1" & + wait + done + sleep 2 + + # Each writer caches its own extent, so a lost update shows on either reader. + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at_on_vm 1 "first_write_$i.bin" "B" "$offset1" 4096 || lost=$((lost + 1)) + done + if [[ $lost -gt 0 ]]; then + log_error "$lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log_success "All concurrent first writes survived" + return 0 +} + test_concurrent_same_file_writes() { log_info "Testing concurrent same-file writes (2 VMs)..." ensure_cluster_healthy 2 @@ -2874,6 +2911,7 @@ main() { run_test "Long-Running Stability" test_stability # Run concurrent same-file write tests (using pre-extension pattern) + run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Concurrent Same-File Writes (2 VMs)" test_concurrent_same_file_writes run_test "Concurrent Same-File Writes (3 VMs)" test_concurrent_same_file_writes_3vms run_test "Mixed Size Writes" test_mixed_size_writes From ff31c15de7a0eb0355ab36c8a55f5e42ccd85f31 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 09:19:03 -0500 Subject: [PATCH 04/18] test: verify first-write lost updates from a restarted reader The VM runner deletes every file before each test, so new files reuse inode numbers and a node that read earlier files serves stale cached extents. That made the VM check fail even though no write was lost. Co-Authored-By: Claude Opus 5.5 --- tests/docker/scripts/run-tests.sh | 5 ++++- tests/vm/scripts/run-vm-tests.sh | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 4acb3212..0136cc9b 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -1874,7 +1874,10 @@ test_concurrent_first_write_same_block() { write_pattern_at "$writer1" "first_write_$i.bin" B "$size" "$offset1" & wait done - sleep 2 + + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + restart_node "$reader" + wait_node_healthy "$reader" 30 || return 1 local lost=0 for i in $(seq 1 $files); do diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index d9018289..c35f2bd0 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2353,7 +2353,12 @@ test_concurrent_first_write_same_block() { done sleep 2 - # Each writer caches its own extent, so a lost update shows on either reader. + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + stop_vm 1 + start_vm 1 false + wait_vm_healthy 1 || return 1 + setup_goclusterfs 1 false + local lost=0 for i in $(seq 1 $files); do verify_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$offset0" 4096 || lost=$((lost + 1)) From f6ddae8c383bd7b8753a1b284d283e354a48c768 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 10:49:38 -0500 Subject: [PATCH 05/18] test(lock): age records through a test helper now that writeRecord is gone Co-Authored-By: Claude Opus 5.5 --- internal/lock/safety_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 7c32e451..11a88aa6 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -322,15 +322,7 @@ func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { writer := newTestNode(dev, 0, 1<<20, 3) requireAcquired(t, leaked, LockTypeInode, 2, LockModeShared) - recordID := leaked.hashToRecord(LockTypeInode, 2) - rec, err := leaked.readRecord(recordID) - if err != nil { - t.Fatalf("readRecord: %v", err) - } - rec.AcquireTime = time.Now().Add(-StaleThreshold + 2*time.Second).Unix() - if err := leaked.writeRecord(recordID, rec); err != nil { - t.Fatalf("writeRecord: %v", err) - } + ageRecord(t, leaked, LockTypeInode, 2, StaleThreshold-2*time.Second) requireAcquired(t, reader, LockTypeInode, 2, LockModeShared) if err := reader.Release(LockTypeInode, 2); err != nil { From 39b2046d86b14d8e33bcc4cccc69bbf1c49c0a37 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:53:11 -0500 Subject: [PATCH 06/18] fix(lock): stop same-node writers waiting on their own readers; renew busy records An exclusive request waited for this node's shared holders of the record. inode.Table holds its mutex across Acquire, and a directory listing holds the directory's shared lock while it reads the table, so creating a file while the directory was listed hung the whole mount. Exclusive requests now join the node's existing hold, as they did before this branch. The record on disk stays shared, so other nodes can read but not write meanwhile. Same-node joins never touched disk, so a record kept busy by overlapping keys kept its first AcquireTime and was taken over as stale after 15s while still held. A key joining a hold older than renewAfter now rewrites AcquireTime. Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 2 + internal/inode/lock_test.go | 60 ++++++++++++++++++++++++++ internal/lock/lock.go | 83 ++++++++++++++++++++++++------------ internal/lock/safety_test.go | 51 ++++++++++------------ tests/test-manifest.yaml | 23 ++++++---- 5 files changed, 155 insertions(+), 64 deletions(-) create mode 100644 internal/inode/lock_test.go diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 095e12a0..8a8f188e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -246,6 +246,8 @@ Common causes of a stall: **Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. +**Locks share records.** The lock region has 256 records, and every lock maps to one of them, so unrelated files and the block bitmap can share a record. A node holding one of those locks blocks other nodes from all of them. If two nodes each hold one record and wait for the other's, both wait until one record is 15 seconds old; the takeover then lets both proceed, with the same risk as a stall. This needs nested locks whose records cross, roughly 1 in 65,000 pairs of overlapping operations. + **Safe operating limits:** | Limit | Why | diff --git a/internal/inode/lock_test.go b/internal/inode/lock_test.go new file mode 100644 index 00000000..137a51b1 --- /dev/null +++ b/internal/inode/lock_test.go @@ -0,0 +1,60 @@ +package inode + +import ( + "context" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/lock" +) + +// A directory listing holds the directory's shared lock and then reads the inode +// table, while Write holds the table mutex and asks for the exclusive lock. +func TestWriteDoesNotHangBehindSameNodeReader(t *testing.T) { + dev, cleanup := createTestDevice(t, 16<<20) + defer cleanup() + var uuid [16]byte + copy(uuid[:], "node-a") + lm := lock.NewManager(dev, 8<<20, 1<<20, uuid, 1) + table := NewTable(dev, 0, 4<<20, lm, uuid) + ctx := context.Background() + dir, err := table.Alloc(ctx) + if err != nil { + t.Fatalf("Alloc: %v", err) + } + key := uint64(dir.InodeNum) + + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + t.Fatalf("Acquire shared: %v", err) + } + writeDone := make(chan error, 1) + go func() { + wctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + writeDone <- table.Write(wctx, dir) + }() + time.Sleep(50 * time.Millisecond) + + readDone := make(chan error, 1) + go func() { + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + readDone <- err + return + } + _, err := table.Read(dir.InodeNum) + lm.Release(lock.LockTypeInode, key) + readDone <- err + }() + lm.Release(lock.LockTypeInode, key) + + for name, done := range map[string]chan error{"Write": writeDone, "listing": readDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("%s: %v", name, err) + } + case <-time.After(2 * time.Second): + t.Fatalf("%s did not finish: same-node reader and writer deadlocked", name) + } + } +} diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 270bd483..192fe7c4 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -239,13 +239,14 @@ type heldLock struct { } type heldRecord struct { - mode LockMode - keys int + mode LockMode + keys int + renewed time.Time } -func covers(held, want LockMode) bool { - return held == LockModeExclusive || held == want -} +// renewAfter is the hold age after which a key joining the record rewrites +// AcquireTime, so a record kept busy by overlapping keys never looks stale. +var renewAfter = StaleThreshold / 3 // Errors var ( @@ -330,7 +331,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin gid := getGoroutineID() m.mu.Lock() - if m.reuseHeldKeyLocked(key, mode, gid) { + if m.reuseHeldKeyLocked(key, gid) { m.mu.Unlock() return nil } @@ -395,24 +396,28 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // attemptAcquire makes one acquisition attempt. added reports whether this // call registered the key rather than joining another goroutine's hold. +// +// A key whose record this node already holds joins that hold in any mode. Exclusive +// requests never wait for this node's readers, who may need a mutex the writer holds; +// meanwhile the record on disk stays shared, so other nodes can read but not write. func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { gate := m.recordGate(recordID) gate.Lock() defer gate.Unlock() m.mu.Lock() - if m.reuseHeldKeyLocked(key, mode, gid) { - m.mu.Unlock() - return true, false, nil - } if rec, ok := m.records[recordID]; ok { - if !covers(rec.mode, mode) { - selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + renew := time.Since(rec.renewed) > renewAfter + m.mu.Unlock() + if renew { + m.renewRecord(recordID, rec) + } + m.mu.Lock() + if held, ok := m.heldLocks[key]; ok { + held.refCount++ + held.goroutines[gid]++ m.mu.Unlock() - if selfDeadlock { - return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) - } - return false, false, nil + return true, false, nil } rec.keys++ m.addHeldKeyLocked(key, recordID, mode, gid) @@ -427,15 +432,15 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi } m.mu.Lock() - m.records[recordID] = &heldRecord{mode: mode, keys: 1} + m.records[recordID] = &heldRecord{mode: mode, keys: 1, renewed: time.Now()} m.addHeldKeyLocked(key, recordID, mode, gid) m.mu.Unlock() return true, true, nil } -func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { +func (m *Manager) reuseHeldKeyLocked(key lockKey, gid uint64) bool { held, ok := m.heldLocks[key] - if !ok || !covers(m.records[held.RecordID].mode, mode) { + if !ok || time.Since(m.records[held.RecordID].renewed) > renewAfter { return false } held.refCount++ @@ -452,15 +457,6 @@ func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, } } -func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { - for _, held := range m.heldLocks { - if held.RecordID == recordID && held.goroutines[gid] > 0 { - return true - } - } - return false -} - func (m *Manager) recordGate(recordID uint64) *sync.Mutex { m.mu.Lock() defer m.mu.Unlock() @@ -663,6 +659,37 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } +// renewRecord rewrites AcquireTime on a record this node holds. It runs under +// the record gate. +func (m *Manager) renewRecord(recordID uint64, rec *heldRecord) { + for { + record, err := m.readRecord(recordID) + if err != nil { + log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) + return + } + ours := record.Mode == rec.mode && (rec.mode == LockModeShared || record.OwnerUUID == m.nodeUUID) + if !ours { + log.Printf("[WARN] lock: record %d was taken over while this node held it", recordID) + return + } + renewed := *record + renewed.AcquireTime = time.Now().Unix() + renewed.Sequence++ + ok, err := m.casRecord(recordID, record, &renewed) + if err != nil { + log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) + return + } + if ok { + m.mu.Lock() + rec.renewed = time.Now() + m.mu.Unlock() + return + } + } +} + // releaseRecord releases this node's hold on a lock record. // After successful release, caches the new sequence number to enable // optimistic acquisition on the next lock attempt. diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 11a88aa6..9677f246 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -153,46 +153,41 @@ func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) } -func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { +func TestExclusiveJoinsSameNodeSharedHold(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) requireAcquired(t, a, LockTypeInode, 9, LockModeShared) - - writer := func() error { - errc := make(chan error, 1) - go func() { - err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) - if err == nil { - err = a.Release(LockTypeInode, 9) - } - errc <- err - }() - return <-errc - } - - if err := writer(); !errors.Is(err, ErrLockTimeout) { - t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) - } - if err := a.Release(LockTypeInode, 9); err != nil { - t.Fatalf("Release shared: %v", err) - } - if err := writer(); err != nil { - t.Fatalf("exclusive Acquire after readers left: %v", err) + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + if err := <-errc; err != nil { + t.Fatalf("exclusive Acquire while this node reads the record: %v", err) } } -func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { +func TestJoiningKeyRenewsBusyRecord(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - a := newTestNode(dev, 0, 1<<20, 1) + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + defer func(d time.Duration) { renewAfter = d }(renewAfter) + renewAfter = 0 - requireAcquired(t, a, LockTypeInode, 9, LockModeShared) - err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) - if !errors.Is(err, ErrDeadlockRisk) { - t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 1, pastStale) + requireAcquired(t, a, LockTypeInode, 2, LockModeExclusive) + if err := a.Release(LockTypeInode, 1); err != nil { + t.Fatalf("Release: %v", err) } + + requireBlocked(t, b, LockTypeInode, 3, LockModeExclusive) } func TestStaleLockIsReclaimedFromPausedHolder(t *testing.T) { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index b5da29c3..b7c260c9 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1370,19 +1370,26 @@ scenarios: environments: go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases - - id: lock-exclusive-waits-same-node-readers - name: Exclusive waits for same-node readers - description: An exclusive request does not piggyback on a shared hold from the same node + - id: lock-exclusive-joins-same-node-readers + name: Exclusive joins same-node readers + description: An exclusive request on a node that already reads the record joins that hold instead of waiting for its own readers priority: go_only environments: - go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + go: internal/lock/safety_test.go::TestExclusiveJoinsSameNodeSharedHold - - id: lock-upgrade-same-goroutine - name: Shared to exclusive upgrade rejected - description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + - id: lock-busy-record-renewed + name: Busy record renewed on join + description: A key joining a record this node has held past renewAfter rewrites its AcquireTime, so a busy node does not lose the record as stale priority: go_only environments: - go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + go: internal/lock/safety_test.go::TestJoiningKeyRenewsBusyRecord + + - id: inode-write-same-node-reader + name: Inode write does not hang behind same-node reader + description: Table.Write on a directory finishes while another goroutine on the node holds the directory's shared lock and reads the inode table + priority: go_only + environments: + go: internal/inode/lock_test.go::TestWriteDoesNotHangBehindSameNodeReader - id: lock-stale-shared-reclaim name: Stale shared lock reclaim drops dead readers From 91169e8f1720daf6e58ea2d1d3f151a63b1e9e27 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 12:06:25 -0500 Subject: [PATCH 07/18] fix(fs): drop the inode lock before freeing blocks and releasing tree reservations The bitmap lock comes before the inode lock, so frees made while holding it fail the lock order check. Tree reservations released under the inode lock parked their blocks in an in-memory list that unmount drops: appends merging into a full inline extent leaked one block each. writeFailed freed the write's blocks the same way. Both now run after the inode lock is released. writeInternal released the inode lock it took for the size update, then released it again in its deferred cleanup. If another goroutine on the node had joined that lock in between, the second release dropped its hold. All releases now go through one helper that clears lockHeld. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 60 ++++++++++++---------- tests/integration/tree_reservation_test.go | 42 +++++++++++++++ tests/test-manifest.yaml | 11 ++++ tests/vm/scripts/run-vm-tests.sh | 2 +- 4 files changed, 88 insertions(+), 27 deletions(-) create mode 100644 tests/integration/tree_reservation_test.go diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index fec5f984..b8fdf1fe 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2075,36 +2075,42 @@ func retryOnTreeConflict(err error) error { // allocateBlockLocked maps a new data block at blockNum and returns the inode // re-read under the inode lock, which stays held on success. The bitmap comes // before the inode in lock order, so the data block and any extent-tree blocks -// are allocated before the lock is taken. -func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, error) { +// are allocated before the lock is taken. Call the returned release after +// dropping the inode lock. +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, func(), error) { blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) if err != nil { - return nil, nil, syscall.ENOSPC + return nil, nil, nil, syscall.ENOSPC } ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) if err != nil { f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, syscall.ENOSPC + return nil, nil, nil, syscall.ENOSPC } - defer releaseTree() if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, syscall.EIO + releaseTree() + return nil, nil, nil, syscall.EIO } - fail := func(errno error) (*inode.Inode, *extent.Extent, error) { + var fresh *inode.Inode + fail := func(errno error) (*inode.Inode, *extent.Extent, func(), error) { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + if fresh != nil { + f.fs.extentMgr.DiscardPending(ctx, fresh) + } f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, errno + releaseTree() + return nil, nil, nil, errno } - fresh, err := f.fs.inodeTbl.Read(f.inodeNum) + fresh, err = f.fs.inodeTbl.Read(f.inodeNum) if err != nil { + fresh = nil return fail(syscall.EIO) } // Insert fails if another node mapped this block, or grew the tree past the reservation, meanwhile. if err := f.fs.extentMgr.Insert(treeCtx, fresh, ext); err != nil { - f.fs.extentMgr.DiscardPending(ctx, fresh) switch { case errors.Is(err, extent.ErrExtentTreeFull): return fail(syscall.EFBIG) @@ -2115,7 +2121,7 @@ func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockN } fresh.AllocatedBlocks += ext.Length fresh.WrittenBlocks += ext.Length - return fresh, ext, nil + return fresh, ext, releaseTree, nil } // writeInternal performs the actual write operation. @@ -2127,9 +2133,18 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i lockHeld := false // ino has changes made under the inode lock that are not on disk yet inoDirty := false - defer func() { + // Tree-block reservations are released after the inode lock, as lock order requires. + var releaseReservations []func() + releaseInodeLock := func() { if lockHeld && f.fs.lockMgr != nil { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + lockHeld = false + } + } + defer func() { + releaseInodeLock() + for _, release := range releaseReservations { + release() } }() @@ -2297,28 +2312,28 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Release inode lock before allocation to respect lock ordering: // bitmap (6) must be acquired before inode (8) - // AllocateBlocks acquires bitmap lock internally if lockHeld && f.fs.lockMgr != nil { // The inode is re-read after allocation, so persist changes made under the lock first. if inoDirty { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { log.Printf("[DEBUG] Write inode %d: persisting inode before allocation failed: %v", f.inodeNum, err) + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) return 0, retryOnTreeConflict(err) } inoDirty = false } - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - lockHeld = false + releaseInodeLock() } if f.fs.lockMgr != nil { - freshIno, newExt, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + freshIno, newExt, releaseTree, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) if allocErr != nil { log.Printf("[DEBUG] Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) return 0, allocErr } lockHeld = true + releaseReservations = append(releaseReservations, releaseTree) ino = freshIno inoDirty = true ext = newExt @@ -2569,9 +2584,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Write inode %d: re-read inode failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, syscall.EIO } // ino was read before the lock, so it can miss another node's extents. @@ -2634,6 +2646,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // On a tree conflict the blocks allocated here are unreferenced. Close the // intent first so journal replay cannot map them again, then free them. writeFailed := func(err error) error { + // The bitmap lock comes before the inode lock, so free only after dropping it. + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) if !errors.Is(err, extent.ErrTreeConflict) { return syscall.EIO @@ -2665,9 +2679,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { log.Printf("[DEBUG] Write inode %d: WriteWithLockHeld failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, writeFailed(err) } } else { @@ -2679,10 +2690,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i inodeWritten = true - // Release the lock we acquired for size update - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } + releaseInodeLock() for _, cow := range cowDone { newCount, decErr := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock) diff --git a/tests/integration/tree_reservation_test.go b/tests/integration/tree_reservation_test.go new file mode 100644 index 00000000..7b5c3787 --- /dev/null +++ b/tests/integration/tree_reservation_test.go @@ -0,0 +1,42 @@ +package integration + +import ( + "syscall" + "testing" +) + +// TestTreeReservationsDoNotLeakBlocks appends to a file whose two inline extents +// are full, so each append reserves an extent-tree block and then merges into the +// last extent without using it. Unused reservations must go back to the bitmap, +// not to an in-memory list that unmount drops, so the whole volume is writable +// again once the file is deleted. +func TestTreeReservationsDoNotLeakBlocks(t *testing.T) { + mfs := NewMountedFS(t, 64) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + // 128KB at block 0 keeps the file off the SFB path; block 2 makes the second extent. + if err := mfs.WriteFileAt("append.bin", chunk(1, 128*1024), 0); err != nil { + t.Fatalf("write at block 0 failed: %v", err) + } + for blk := 2; blk <= 7; blk++ { + if err := mfs.WriteFileAt("append.bin", chunk(byte(blk), 64*1024), int64(blk*mb)); err != nil { + t.Fatalf("write at block %d failed: %v", blk, err) + } + } + remount(t, mfs) + if err := mfs.Remove("append.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + + var st syscall.Statfs_t + if err := syscall.Statfs(mfs.MountPoint, &st); err != nil { + t.Fatalf("Statfs failed: %v", err) + } + // Every data block except the root directory's. + fill := int(st.Blocks-1) * mb + if err := mfs.WriteFile("fill.bin", make([]byte, fill)); err != nil { + t.Fatalf("writing %d MB into an empty volume of %d blocks failed: %v", fill/mb, st.Blocks, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index b7c260c9..76036225 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -313,6 +313,17 @@ scenarios: docker: run-tests.sh::test_concurrent_first_write_same_block vm: run-vm-tests.sh::test_concurrent_first_write_same_block + # Docker/VM TODO: single-node behavior that needs a remount and a full-volume + # write; the Go integration test runs the real mount binary already. + - id: tree-reservation-no-leak + name: Extent-tree reservations do not leak blocks + description: Appends that merge into a full inline extent return their unused tree-block reservation, so the whole volume is writable after the file is deleted and the FS remounted + priority: medium + environments: + go: tree_reservation_test.go::TestTreeReservationsDoNotLeakBlocks + docker: TODO + vm: TODO + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index c35f2bd0..4bac69f4 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2329,7 +2329,6 @@ test_stability() { # Concurrent Same-File Write Tests (using pre-extension pattern) # ============================================================================= -# Test 10: Two VMs write to same file at different offsets # Two VMs write different ranges of each new file's first block at once. # Without the inode lock the later inode write drops the other VM's extent. test_concurrent_first_write_same_block() { @@ -2372,6 +2371,7 @@ test_concurrent_first_write_same_block() { return 0 } +# Test 10: Two VMs write to same file at different offsets test_concurrent_same_file_writes() { log_info "Testing concurrent same-file writes (2 VMs)..." ensure_cluster_healthy 2 From 17d6982ebd255ab564e5a4ca11f71b44c64aa54e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:09:37 -0500 Subject: [PATCH 08/18] fix(lock): stop hash collisions and same-node races from creating double holders The lock region has 256 records in the production layout, and tryAcquire treated a record held for a different resource as free. Any node could take over a held lock by touching a resource that hashed to the same record. Other double-holder windows fixed here: - two goroutines on one node acquiring the same key at once left the on-disk record cleared while both believed they held it - releasing a shared lock cleared the record for all readers when the releasing node was the last one to join - an exclusive request piggybacked on a same-node shared hold - reclaiming a stale shared record kept the dead readers' count - a release whose CAS lost a race returned nil without releasing Records now act as lock stripes: a held record blocks every resource that hashes to it, a node can hold several keys on one record, and a per-record gate serializes this node's on-disk acquire and release. Co-Authored-By: Claude Opus 5.5 --- internal/lock/lock.go | 314 +++++++++++++++++++--------------- internal/lock/safety_test.go | 315 +++++++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 81 +++++++++ 3 files changed, 575 insertions(+), 135 deletions(-) create mode 100644 internal/lock/safety_test.go diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 042fed5f..9405ab3c 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -206,6 +206,11 @@ type Manager struct { nodeSlot int // This node's slot heldLocks map[lockKey]*heldLock + // Several keys can hash to one record; the record stays held until the last key is released. + records map[uint64]*heldRecord + // gates serialize this node's on-disk operations per record. + gates map[uint64]*sync.Mutex + // Callback for cache invalidation onAcquire OnAcquireFunc @@ -233,6 +238,15 @@ type heldLock struct { goroutines map[uint64]int // Goroutine ID -> reference count for that goroutine } +type heldRecord struct { + mode LockMode + keys int +} + +func covers(held, want LockMode) bool { + return held == LockModeExclusive || held == want +} + // Errors var ( ErrLockTimeout = errors.New("lock acquisition timed out") @@ -251,6 +265,8 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(HintCacheSize), } } @@ -265,6 +281,8 @@ func NewManagerWithHintCacheSize(dev *storage.Device, offset, size uint64, nodeU nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(hintCacheSize), } } @@ -303,33 +321,25 @@ func (m *Manager) HintStats() (hits, misses uint64, hitRate float64) { // subsequent goroutines increment a local reference count. This allows // concurrent FUSE operations from the same node to share locks efficiently // while still providing cross-node serialization. +// +// Resources that hash to the same record share it: a node holding the record +// for one resource excludes other nodes from every resource in that record. func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uint64, mode LockMode) error { key := lockKey{Type: lockType, ResourceID: resourceID} - + recordID := m.hashToRecord(lockType, resourceID) gid := getGoroutineID() m.mu.Lock() - // Check if we already hold this lock (same node) - just increment refCount - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - reusing existing lock (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) + if m.reuseHeldKeyLocked(key, mode, gid) { m.mu.Unlock() return nil } - - // Check lock ordering to prevent deadlocks if err := m.checkLockOrder(lockType); err != nil { m.mu.Unlock() return err } m.mu.Unlock() - // Calculate record position (hash-based) - recordID := m.hashToRecord(lockType, resourceID) retryCount := 0 startTime := time.Now() @@ -338,63 +348,30 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin for { // Always attempt acquire first, then check context after failure // This ensures we try at least once even if context is nearly expired - acquired, err := m.tryAcquire(recordID, lockType, resourceID, mode) + held, added, err := m.attemptAcquire(key, recordID, mode, gid) if err != nil { - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - tryAcquire error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - attempt error: %v", lockType, resourceID, err) return err } - if acquired { - m.mu.Lock() - // Double-check in case another goroutine acquired while we were trying - if held, ok := m.heldLocks[key]; ok { - // Another goroutine beat us - just increment refCount - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - late arrival, reusing (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) - m.mu.Unlock() - // Release the on-disk lock we just acquired since we'll share the existing one - m.releaseRecord(recordID) - return nil - } - m.heldLocks[key] = &heldLock{ - Mode: mode, - RecordID: recordID, - refCount: 1, - goroutines: map[uint64]int{gid: 1}, - } - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired distributed lock (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) + if held { + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) // Trigger cache invalidation callback for exclusive locks. // We do this before returning so the caller reads fresh data. // Only exclusive locks trigger invalidation since shared locks // are for reading and can safely use cached values. + m.mu.Lock() onAcquire := m.onAcquire m.mu.Unlock() - - if onAcquire != nil && mode == LockModeExclusive { + if added && onAcquire != nil && mode == LockModeExclusive { onAcquire(lockType, resourceID, mode) } return nil } - // Check again if another goroutine acquired while we were waiting - m.mu.Lock() - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - piggyback on existing (refCount=%d)", lockType, resourceID, held.refCount) - m.mu.Unlock() - return nil - } - m.mu.Unlock() - retryCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - loop iteration, retryCount now %d", lockType, resourceID, retryCount) - // Check context cancellation, but allow at least 3 retries for same-node - // CAS races where the other goroutine is still adding to heldLocks + // Check context cancellation, but allow at least 3 retries if retryCount >= 3 { select { case <-ctx.Done(): @@ -416,6 +393,85 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin } } +// attemptAcquire makes one acquisition attempt. added reports whether this +// call registered the key rather than joining another goroutine's hold. +func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + if m.reuseHeldKeyLocked(key, mode, gid) { + m.mu.Unlock() + return true, false, nil + } + if rec, ok := m.records[recordID]; ok { + if !covers(rec.mode, mode) { + selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + m.mu.Unlock() + if selfDeadlock { + return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) + } + return false, false, nil + } + rec.keys++ + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil + } + m.mu.Unlock() + + acquired, err := m.tryAcquire(recordID, key.Type, key.ResourceID, mode) + if err != nil || !acquired { + return false, false, err + } + + m.mu.Lock() + m.records[recordID] = &heldRecord{mode: mode, keys: 1} + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil +} + +func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { + held, ok := m.heldLocks[key] + if !ok || !covers(m.records[held.RecordID].mode, mode) { + return false + } + held.refCount++ + held.goroutines[gid]++ + return true +} + +func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, gid uint64) { + m.heldLocks[key] = &heldLock{ + Mode: mode, + RecordID: recordID, + refCount: 1, + goroutines: map[uint64]int{gid: 1}, + } +} + +func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { + for _, held := range m.heldLocks { + if held.RecordID == recordID && held.goroutines[gid] > 0 { + return true + } + } + return false +} + +func (m *Manager) recordGate(recordID uint64) *sync.Mutex { + m.mu.Lock() + defer m.mu.Unlock() + gate, ok := m.gates[recordID] + if !ok { + gate = &sync.Mutex{} + m.gates[recordID] = gate + } + return gate +} + // Release releases a held lock. // With reference counting, this decrements the local count and only releases // the distributed lock when the last goroutine releases. @@ -431,36 +487,51 @@ func (m *Manager) Release(lockType LockType, resourceID uint64) error { return ErrNotLockOwner } - // Decrement goroutine-specific count - if held.goroutines != nil { - held.goroutines[gid]-- - if held.goroutines[gid] <= 0 { - delete(held.goroutines, gid) - } + held.goroutines[gid]-- + if held.goroutines[gid] <= 0 { + delete(held.goroutines, gid) } // Decrement overall reference count held.refCount-- if held.refCount > 0 { - // Other goroutines still using the lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - decremented refCount to %d", lockType, resourceID, gid, held.refCount) m.mu.Unlock() return nil } - // Last reference - release the distributed lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - last ref, releasing distributed lock", lockType, resourceID, gid) delete(m.heldLocks, key) recordID := held.RecordID m.mu.Unlock() - err := m.releaseRecord(recordID) + err := m.dropRecordKey(recordID) if err != nil { - log.Printf("[DEBUG] lock.Release: type=%d res=%d - releaseRecord error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Release: type=%d res=%d - release error: %v", lockType, resourceID, err) } return err } +func (m *Manager) dropRecordKey(recordID uint64) error { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + rec, ok := m.records[recordID] + if !ok { + m.mu.Unlock() + return ErrNotLockOwner + } + rec.keys-- + if rec.keys > 0 { + m.mu.Unlock() + return nil + } + delete(m.records, recordID) + m.mu.Unlock() + + return m.releaseRecord(recordID, rec.mode) +} + // tryAcquire attempts a single lock acquisition. // First tries an optimistic path using cached sequence hints, then falls back // to reading the current record if the hint is stale. @@ -499,40 +570,26 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint now := time.Now() - // Check if lock is free or can be acquired - canAcquire := false - reason := "unknown" + // A record held for a colliding resource is still held; taking it over lets two nodes hold it. + var sharedCount uint32 + reason := "" switch { case record.Mode == LockModeNone: - // Lock is free - canAcquire = true reason = "free" - case record.Type != lockType || record.ResourceID != resourceID: - // Hash collision - different resource, treat as free - // In production, we'd use a more sophisticated collision resolution - canAcquire = true - reason = "hash_collision" - case m.isLockStale(record): - // Lock is stale (owner died) - canAcquire = true + // Owner presumed dead; its shared holders are gone too. reason = "stale" case mode == LockModeShared && record.Mode == LockModeShared: - // Adding another shared holder - canAcquire = true reason = "shared_add" + sharedCount = record.SharedCount - case record.OwnerUUID == m.nodeUUID: - // Same node owns the lock - this can happen due to a race between - // Release clearing heldLocks and releaseRecord clearing on-disk. - // Safe to re-acquire since the previous holder from this node is gone. - canAcquire = true - reason = "same_node_reacquire" - } + case record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID: + // Leftover from a failed release or an earlier mount; this node holds no key on it. + reason = "leftover_from_this_node" - if !canAcquire { + default: log.Printf("[DEBUG] lock.tryAcquire: type=%d res=%d - cannot acquire, held by node=%x mode=%d", lockType, resourceID, record.OwnerUUID[:8], record.Mode) return false, nil } @@ -551,7 +608,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint Sequence: record.Sequence + 1, } if mode == LockModeShared { - newRecord.SharedCount = record.SharedCount + 1 + newRecord.SharedCount = sharedCount + 1 } // CAS the record @@ -603,58 +660,45 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } -// releaseRecord releases a lock record. +// releaseRecord releases this node's hold on a lock record. // After successful release, caches the new sequence number to enable // optimistic acquisition on the next lock attempt. -func (m *Manager) releaseRecord(recordID uint64) error { - record, err := m.readRecord(recordID) - if err != nil { - return err - } - - // Verify we own it - if record.OwnerUUID != m.nodeUUID { - // For shared locks, just decrement count - if record.Mode == LockModeShared && record.SharedCount > 1 { - newRecord := *record - newRecord.SharedCount-- - newRecord.Sequence++ - success, err := m.casRecord(recordID, record, &newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeShared, // Still shared (decremented) - UpdateAt: time.Now(), - }) - } - return nil +func (m *Manager) releaseRecord(recordID uint64, mode LockMode) error { + for { + record, err := m.readRecord(recordID) + if err != nil { + return err } - return ErrNotLockOwner - } - // Clear the record - newRecord := &Record{ - Sequence: record.Sequence + 1, - } + var newRecord *Record + switch { + case mode == LockModeExclusive && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID, + mode == LockModeShared && record.Mode == LockModeShared && record.SharedCount <= 1: + newRecord = &Record{Sequence: record.Sequence + 1} + case mode == LockModeShared && record.Mode == LockModeShared: + // OwnerUUID names only the latest reader, so readers release by count. + next := *record + next.SharedCount-- + next.Sequence++ + newRecord = &next + default: + // Another node reclaimed the record after it went stale. + return ErrNotLockOwner + } - success, err := m.casRecord(recordID, record, newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - // Mode is None since lock is now free - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeNone, - UpdateAt: time.Now(), - }) + success, err := m.casRecord(recordID, record, newRecord) + if err != nil { + return err + } + if success { + m.hintCache.Put(recordID, &lockHint{ + Sequence: newRecord.Sequence, + Mode: newRecord.Mode, + UpdateAt: time.Now(), + }) + return nil + } } - return nil } // isLockStale checks if a lock's owner is dead. diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go new file mode 100644 index 00000000..29267588 --- /dev/null +++ b/internal/lock/safety_test.go @@ -0,0 +1,315 @@ +package lock + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/storage" +) + +func newTestNode(dev *storage.Device, offset, size uint64, n int) *Manager { + var uuid [16]byte + copy(uuid[:], fmt.Sprintf("test-node-%06d", n)) + return NewManager(dev, offset, size, uuid, n) +} + +func acquireWithin(m *Manager, d time.Duration, lt LockType, id uint64, mode LockMode) error { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + return m.Acquire(ctx, lt, id, mode) +} + +func requireBlocked(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + err := acquireWithin(m, 50*time.Millisecond, lt, id, mode) + if err == nil { + m.Release(lt, id) + t.Fatalf("node %d acquired type=%d id=%d mode=%d while another holder should exclude it", m.nodeSlot, lt, id, mode) + } + if !errors.Is(err, ErrLockTimeout) { + t.Fatalf("Acquire: got %v, want ErrLockTimeout", err) + } +} + +func requireAcquired(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + if err := acquireWithin(m, time.Second, lt, id, mode); err != nil { + t.Fatalf("node %d Acquire type=%d id=%d mode=%d: %v", m.nodeSlot, lt, id, mode, err) + } +} + +// ageRecord rewrites the lock's record as if it was taken age ago and never renewed. +func ageRecord(t *testing.T, m *Manager, lt LockType, id uint64, age time.Duration) { + t.Helper() + recordID := m.hashToRecord(lt, id) + rec, err := m.readRecord(recordID) + if err != nil { + t.Fatalf("readRecord: %v", err) + } + rec.AcquireTime = time.Now().Add(-age).Unix() + buf := storage.GetBuffer(RecordSize) + defer storage.PutBuffer(buf) + copy(buf.Bytes(), serializeRecord(rec)) + if _, err := m.dev.WriteAt(buf.Bytes(), int64(m.offset)+int64(recordID)*RecordSize); err != nil { + t.Fatalf("write record: %v", err) + } +} + +const pastStale = StaleThreshold + 2*time.Second + +func TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + requireBlocked(t, b, LockTypeInode, 2, LockModeExclusive) +} + +func TestNodeHoldsCollidingResourcesUntilLastRelease(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + requireAcquired(t, a, LockTypeInode, 7, LockModeExclusive) + if err := a.Release(LockTypeInode, 7); err != nil { + t.Fatalf("Release inode: %v", err) + } + requireBlocked(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + + if err := a.Release(LockTypeBlockBitmap, 0); err != nil { + t.Fatalf("Release bitmap: %v", err) + } + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) +} + +func TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + const workers = 4 + for iter := 0; iter < 30; iter++ { + start := make(chan struct{}) + release := make(chan struct{}) + var acquired, done sync.WaitGroup + errs := make(chan error, workers) + acquired.Add(workers) + done.Add(workers) + for w := 0; w < workers; w++ { + go func() { + defer done.Done() + <-start + err := acquireWithin(a, time.Second, LockTypeInode, 42, LockModeExclusive) + acquired.Done() + if err != nil { + errs <- err + return + } + <-release + if err := a.Release(LockTypeInode, 42); err != nil { + errs <- err + } + }() + } + close(start) + acquired.Wait() + requireBlocked(t, b, LockTypeInode, 42, LockModeExclusive) + close(release) + done.Wait() + close(errs) + for err := range errs { + t.Fatalf("iteration %d: %v", iter, err) + } + } +} + +func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 5, LockModeShared) + requireAcquired(t, b, LockTypeInode, 5, LockModeShared) + if err := b.Release(LockTypeInode, 5); err != nil { + t.Fatalf("b Release: %v", err) + } + requireBlocked(t, c, LockTypeInode, 5, LockModeExclusive) + + if err := a.Release(LockTypeInode, 5); err != nil { + t.Fatalf("a Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) +} + +func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + + writer := func() error { + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + return <-errc + } + + if err := writer(); !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) + } + if err := a.Release(LockTypeInode, 9); err != nil { + t.Fatalf("Release shared: %v", err) + } + if err := writer(); err != nil { + t.Fatalf("exclusive Acquire after readers left: %v", err) + } +} + +func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) + if !errors.Is(err, ErrDeadlockRisk) { + t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + } +} + +func TestStaleLockIsReclaimedFromPausedHolder(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + ageRecord(t, a, LockTypeBlockBitmap, 0, pastStale) + + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + if held := a.HeldLocks(); len(held) != 1 { + t.Fatalf("paused holder's view: got %v, want it to still list the lock (no fencing)", held) + } +} + +func TestPausedHolderReleaseKeepsReclaimedLock(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 3, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 3, pastStale) + requireAcquired(t, b, LockTypeInode, 3, LockModeExclusive) + + if err := a.Release(LockTypeInode, 3); !errors.Is(err, ErrNotLockOwner) { + t.Fatalf("paused holder Release: got %v, want ErrNotLockOwner", err) + } + requireBlocked(t, c, LockTypeInode, 3, LockModeExclusive) +} + +func TestStaleSharedLockReclaimResetsReaderCount(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + dead := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, dead, LockTypeInode, 4, LockModeShared) + ageRecord(t, dead, LockTypeInode, 4, pastStale) + requireAcquired(t, a, LockTypeInode, 4, LockModeShared) + if err := a.Release(LockTypeInode, 4); err != nil { + t.Fatalf("Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 4, LockModeExclusive) +} + +func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { + const ( + guard = 4 * RecordSize + regionSize = 16 * RecordSize + resources = 1000 + cycles = 1500 + ) + dev, cleanup := createTestDevice(t, guard+regionSize+guard) + defer cleanup() + + sentinel := storage.NewAlignedBuffer(guard) + for i := range sentinel.Bytes() { + sentinel.Bytes()[i] = 0xA5 + } + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.WriteAt(sentinel.Bytes(), off); err != nil { + t.Fatalf("write sentinel: %v", err) + } + } + + nodes := []*Manager{ + newTestNode(dev, guard, regionSize, 1), + newTestNode(dev, guard, regionSize, 2), + } + var wg sync.WaitGroup + errs := make(chan error, len(nodes)) + for n, m := range nodes { + wg.Add(1) + go func(n int, m *Manager) { + defer wg.Done() + for i := 0; i < cycles; i++ { + id := uint64((i*7 + n*13) % resources) + lt := LockTypeInode + if i%3 == 0 { + lt = LockTypeExtent + } + if err := acquireWithin(m, 5*time.Second, lt, id, LockModeExclusive); err != nil { + errs <- fmt.Errorf("node %d cycle %d: %w", n, i, err) + return + } + m.Release(lt, id) + } + }(n, m) + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + check := newTestNode(dev, guard, regionSize, 3) + for id := uint64(0); id < resources; id++ { + for _, lt := range []LockType{LockTypeInode, LockTypeExtent} { + requireAcquired(t, check, lt, id, LockModeExclusive) + if err := check.Release(lt, id); err != nil { + t.Fatalf("Release type=%d id=%d: %v", lt, id, err) + } + } + } + + got := storage.NewAlignedBuffer(guard) + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.ReadAt(got.Bytes(), off); err != nil { + t.Fatalf("read sentinel: %v", err) + } + if !bytes.Equal(got.Bytes(), sentinel.Bytes()) { + t.Fatalf("lock manager wrote outside its region near offset %d", off) + } + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index e4a4fc47..91e92ccc 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1345,3 +1345,84 @@ scenarios: priority: go_only environments: go: internal/journal/journal_fuzz_test.go::FuzzJournalScan + + # =========================================================================== + # DISTRIBUTED LOCK SAFETY (internal/lock) + # Two or three lock managers share one device to act as separate nodes. + # =========================================================================== + + # Docker/VM TODO: a shell test would have to freeze a node (docker pause, or + # SIGSTOP on the QEMU process) inside a lock-held section that lasts + # milliseconds, then resume it after StaleThreshold. Shell tests cannot target + # that window, so the Go test simulates the pause by aging the on-disk record. + - id: lock-stale-reclaim-paused-holder + name: Stale lock reclaimed from paused holder + description: A holder stalled past StaleThreshold loses its lock to another node without being told + priority: high + environments: + go: ../../internal/lock/safety_test.go::TestStaleLockIsReclaimedFromPausedHolder + docker: TODO + vm: TODO + + - id: lock-paused-holder-release + name: Paused holder release keeps new holder's lock + description: A late Release from a holder whose lock was reclaimed returns ErrNotLockOwner and leaves the new holder's record intact + priority: go_only + environments: + go: internal/lock/safety_test.go::TestPausedHolderReleaseKeepsReclaimedLock + + - id: lock-colliding-resources-excluded + name: Colliding resources exclude other nodes + description: A held record blocks other nodes from every resource that hashes to it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord + + - id: lock-same-node-colliding-resources + name: One node holds colliding resources + description: A node can hold two resources that share a record; the record stays held until the last one is released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNodeHoldsCollidingResourcesUntilLastRelease + + - id: lock-same-node-concurrent-acquire + name: Concurrent same-node acquire keeps the lock + description: Goroutines on one node acquiring the same lock at once leave it held on disk + priority: go_only + environments: + go: internal/lock/safety_test.go::TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut + + - id: lock-shared-release-by-count + name: Shared lock survives another reader's release + description: Releasing one reader's shared hold does not free the lock for writers while other readers hold it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases + + - id: lock-exclusive-waits-same-node-readers + name: Exclusive waits for same-node readers + description: An exclusive request does not piggyback on a shared hold from the same node + priority: go_only + environments: + go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + + - id: lock-upgrade-same-goroutine + name: Shared to exclusive upgrade rejected + description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + + - id: lock-stale-shared-reclaim + name: Stale shared lock reclaim drops dead readers + description: Reclaiming a stale shared record resets the reader count so it can be fully released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestStaleSharedLockReclaimResetsReaderCount + + - id: lock-region-bounded + name: Lock region stays bounded + description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestLockRegionStaysBoundedAcrossCycles From 001fbd10716a65d08927d7a64567e6b58b7a37ae Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:09:37 -0500 Subject: [PATCH 09/18] docs(operations): document lock stall hazard and safe operating limits Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 591915b3..f006c084 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -229,12 +229,42 @@ rm /mnt/fs/.snapshots/snapshot_name iscsiadm -m node -T -p --logout ``` +### Distributed Lock Limits + +Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, SFB containers, snapshots) through lock records on the shared device. Read this before you run more than one node. + +**How a lock goes stale.** A lock record stores the wall-clock time it was taken. Nothing renews it. Any other node may take over a lock that is older than 15 seconds (`StaleThreshold`), judged by that node's own clock. This is how locks held by a crashed node get freed. + +**The hazard: a stalled node keeps writing.** If a node stops for more than 15 seconds while it holds a lock, another node takes the lock. When the stalled node resumes, it finishes its writes. No write path checks that the lock is still held, and there is no fencing token. The node only learns it lost the lock when it releases it (`ErrNotLockOwner`), and callers ignore that error. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. + +Common causes of a stall: + +- VM pause, suspend, live migration, or a memory snapshot +- `docker pause`, cgroup freezer, `SIGSTOP`, or a debugger on the mount process +- Heavy swapping or a long GC pause +- Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover + +**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. + +**Safe operating limits:** + +| Limit | Why | +|-------|-----| +| Keep node clocks within 1 second (chrony or NTP) | Clock skew S shortens the stale threshold to 15s − S | +| No node may stall for 10 seconds or more while mounted | Leaves margin below the 15s threshold | +| Storage I/O must fail, not queue, within 10 seconds | A queued write that lands after takeover corrupts metadata. Do not use multipath `queue_if_no_path`; lower iSCSI `node.session.timeo.replacement_timeout` (default 120s) | +| Unmount before you pause, migrate or snapshot a node VM | Unmounting releases its locks | + +If a node did stall for more than 15 seconds while mounted (hung-task warnings in `dmesg`, a VM pause, a path failover), stop its mount process before it does more writes, then check the data from a healthy node. + +What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks 15 seconds after it acquired them. + ### Handling Node Failure When a node fails unexpectedly: 1. **Other nodes continue operating** - Distributed locking handles failed node -2. **Lock cleanup** - Stale locks are detected via heartbeat timeout +2. **Lock cleanup** - Other nodes take over the failed node's locks once they are 15 seconds old (lock age, not node heartbeat) 3. **No immediate action required** - System is self-healing **When failed node recovers**: From 4d3c5a89aa94783158e98c063539c0272b52cd73 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 08:21:50 -0500 Subject: [PATCH 10/18] fix(fs): wire the lock manager into mounted filesystems cmd/mount built the FS with NewWithComponents, which had no lock manager parameter, so every inode, extent and SFB lock branch in internal/goclusterfs was dead in production. Two nodes writing a new file's first block at the same time lost one node's data in 11 of 20 files. The dead branches had never run and lost data once enabled: after allocating a block without the inode lock, writeInternal and Allocate re-read the inode under the lock and dropped the new extent. They now map the new block in the fresh inode, or free it and retry if another node mapped that block first. Changes made under the lock are written before the lock is dropped for the next allocation, and the final size update builds on the inode read under the lock. Readers joining a shared lock no longer refresh its age. A reader count leaked by a lost CAS update now goes stale after 15s instead of blocking writers for as long as readers keep arriving; Docker hung on rmdir without this. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 2 +- docs/OPERATIONS.md | 2 +- internal/goclusterfs/fs.go | 123 +++++++++++++++++++++------- internal/lock/lock.go | 5 +- internal/lock/safety_test.go | 27 ++++++ tests/docker/scripts/run-tests.sh | 38 +++++++++ tests/integration/multinode_test.go | 74 +++++++++++++++++ tests/test-manifest.yaml | 16 ++++ tests/vm/scripts/run-vm-tests.sh | 38 +++++++++ 9 files changed, 292 insertions(+), 33 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 7d13bd36..42e7a9db 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -279,7 +279,7 @@ func main() { } // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency - gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, nodeUUID, dataOffset) + gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) // Set up cross-node recovery handler for failed node journal replay recoveryCtx := &journal.RecoveryContext{ diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index f006c084..095e12a0 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -244,7 +244,7 @@ Common causes of a stall: - Heavy swapping or a long GC pause - Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover -**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. +**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. **Safe operating limits:** diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index fac3e201..a25fa508 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -194,11 +194,13 @@ func NewWithComponents( tm *trim.Manager, // May be nil if TRIM is disabled sfbMgr *sfb.Manager, // May be nil if SFB is disabled refcountReg *refcount.Region, // May be nil if snapshots not enabled + lockMgr *lock.Manager, // May be nil for single-node use nodeUUID [16]byte, dataOffset uint64, ) *FS { return &FS{ dev: dev, + lockMgr: lockMgr, bitmapMgr: bm, inodeTbl: inodeTbl, extentMgr: extentMgr, @@ -2095,6 +2097,52 @@ func (f *FS) zeroBlock(physBlock uint64) error { return err } +// allocateBlockLocked maps a new data block at blockNum and returns the inode +// re-read under the inode lock, which stays held on success. The bitmap comes +// before the inode in lock order, so the data block and any extent-tree blocks +// are allocated before the lock is taken. +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, error) { + blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) + if err != nil { + return nil, nil, syscall.ENOSPC + } + ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} + treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) + if err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, syscall.ENOSPC + } + defer releaseTree() + + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, syscall.EIO + } + fail := func(errno error) (*inode.Inode, *extent.Extent, error) { + f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, errno + } + fresh, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + return fail(syscall.EIO) + } + // Insert fails if another node mapped this block, or grew the tree past the reservation, meanwhile. + if err := f.fs.extentMgr.Insert(treeCtx, fresh, ext); err != nil { + f.fs.extentMgr.DiscardPending(ctx, fresh) + switch { + case errors.Is(err, extent.ErrExtentTreeFull): + return fail(syscall.EFBIG) + case errors.Is(err, extent.ErrCorruptTree): + return fail(syscall.EIO) + } + return fail(errRetryWrite) + } + fresh.AllocatedBlocks += ext.Length + fresh.WrittenBlocks += ext.Length + return fresh, ext, nil +} + // writeInternal performs the actual write operation. // Returns errRetryWrite if a concurrent modification is detected and the caller should retry. func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (int, error) { @@ -2102,6 +2150,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Track if we hold the inode lock lockHeld := false + // ino has changes made under the inode lock that are not on disk yet + inoDirty := false defer func() { if lockHeld && f.fs.lockMgr != nil { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) @@ -2205,42 +2255,46 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // bitmap (6) must be acquired before inode (8) // AllocateBlocks acquires bitmap lock internally if lockHeld && f.fs.lockMgr != nil { + // The inode is re-read after allocation, so persist changes made under the lock first. + if inoDirty { + if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { + log.Printf("[DEBUG] Write inode %d: persisting inode before allocation failed: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, retryOnTreeConflict(err) + } + inoDirty = false + } f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) lockHeld = false } - // Allocate block (acquires bitmap lock internally, then releases it) - ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) - if err != nil { - log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) - switch { - case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): - // Concurrent allocation by another node - return 0, errRetryWrite - case errors.Is(err, extent.ErrExtentTreeFull): - return 0, syscall.EFBIG - case errors.Is(err, extent.ErrCorruptTree): - return 0, syscall.EIO - default: - return 0, syscall.ENOSPC - } - } - - // Re-acquire inode lock after allocation if f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to re-acquire lock after allocation: %v", f.inodeNum, err) - return 0, syscall.EIO + freshIno, newExt, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + if allocErr != nil { + log.Printf("[DEBUG] Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) + return 0, allocErr } lockHeld = true - - // Re-read inode after re-acquiring lock to verify state - freshIno, readErr := f.fs.inodeTbl.Read(f.inodeNum) - if readErr != nil { - log.Printf("[DEBUG] Write inode %d: re-read after allocation failed: %v", f.inodeNum, readErr) - return 0, syscall.EIO - } ino = freshIno + inoDirty = true + ext = newExt + } else { + // Allocate block (acquires bitmap lock internally, then releases it) + ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) + if err != nil { + log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) + switch { + case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): + // Concurrent allocation by another node + return 0, errRetryWrite + case errors.Is(err, extent.ErrExtentTreeFull): + return 0, syscall.EFBIG + case errors.Is(err, extent.ErrCorruptTree): + return 0, syscall.EIO + default: + return 0, syscall.ENOSPC + } + } } extOffset = 0 @@ -2315,6 +2369,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } if cowResult.WasShared { + inoDirty = true log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) @@ -2373,6 +2428,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return 0, syscall.EIO } if cleared { + inoDirty = true log.Printf("[DEBUG] Write inode %d: cleared Unwritten flag for block %d", f.inodeNum, blockNum) } @@ -2487,6 +2543,10 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } return 0, syscall.EIO } + // ino was read before the lock, so it can miss another node's extents. + if acquiredLockForSize { + ino = freshIno + } // Use the freshest size from disk, then calculate new size as max(fresh_size, our_write_end) newSize := freshIno.Size if offset > newSize { @@ -3078,12 +3138,15 @@ func (f *File) allocate(ctx context.Context, off uint64, size uint64, mode uint3 } defer f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - // Re-read inode after lock to get latest state - ino, err = f.fs.inodeTbl.Read(f.inodeNum) + // ino holds the new extents, so keep it only if no node wrote the inode since it was read. + fresh, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Allocate inode %d: re-read failed: %v", f.inodeNum, err) return syscall.EIO, nil } + if fresh.MetaGeneration != ino.MetaGeneration { + return 0, extent.ErrStaleTree + } } // Update file size if not KEEP_SIZE and new size is larger diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 9405ab3c..270bd483 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -572,6 +572,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint // A record held for a colliding resource is still held; taking it over lets two nodes hold it. var sharedCount uint32 + acquireTime := now.Unix() reason := "" switch { case record.Mode == LockModeNone: @@ -584,6 +585,8 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint case mode == LockModeShared && record.Mode == LockModeShared: reason = "shared_add" sharedCount = record.SharedCount + // Keep the first reader's time: a count leaked by a lost CAS update must still go stale. + acquireTime = record.AcquireTime case record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID: // Leftover from a failed release or an earlier mount; this node holds no key on it. @@ -604,7 +607,7 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint ResourceID: resourceID, OwnerUUID: m.nodeUUID, OwnerSlot: uint32(m.nodeSlot), - AcquireTime: now.Unix(), + AcquireTime: acquireTime, Sequence: record.Sequence + 1, } if mode == LockModeShared { diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 29267588..7c32e451 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -313,3 +313,30 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { } } } + +func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + leaked := newTestNode(dev, 0, 1<<20, 1) + reader := newTestNode(dev, 0, 1<<20, 2) + writer := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, leaked, LockTypeInode, 2, LockModeShared) + recordID := leaked.hashToRecord(LockTypeInode, 2) + rec, err := leaked.readRecord(recordID) + if err != nil { + t.Fatalf("readRecord: %v", err) + } + rec.AcquireTime = time.Now().Add(-StaleThreshold + 2*time.Second).Unix() + if err := leaked.writeRecord(recordID, rec); err != nil { + t.Fatalf("writeRecord: %v", err) + } + + requireAcquired(t, reader, LockTypeInode, 2, LockModeShared) + if err := reader.Release(LockTypeInode, 2); err != nil { + t.Fatalf("reader Release: %v", err) + } + time.Sleep(3 * time.Second) + + requireAcquired(t, writer, LockTypeInode, 2, LockModeExclusive) +} diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 0f7406fc..4acb3212 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -1852,6 +1852,43 @@ test_concurrent_writes() { return 0 } +# Two nodes write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other node's extent. +test_concurrent_first_write_same_block() { + log "Testing concurrent first writes to the same block from two nodes..." + local writer0="${NODE_ARRAY[0]}" + local writer1="${NODE_ARRAY[1]}" + local reader="${NODE_ARRAY[2]}" + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + exec_on_node "$writer0" touch "/mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at "$writer0" "first_write_$i.bin" A "$size" "$offset0" & + write_pattern_at "$writer1" "first_write_$i.bin" B "$size" "$offset1" & + wait + done + sleep 2 + + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at "$reader" "first_write_$i.bin" A "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at "$reader" "first_write_$i.bin" B "$offset1" 4096 || lost=$((lost + 1)) + done + if [ "$lost" -gt 0 ]; then + log "ERROR: $lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log "All concurrent first writes survived" + return 0 +} + # Test 2: Container kill and recovery test_container_kill_recovery() { log "Testing container kill and recovery..." @@ -2717,6 +2754,7 @@ else run_test "Concurrent File Creation" test_concurrent_file_creation run_test "Concurrent Directory Ops" test_concurrent_directory_ops fi +run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Container Kill Recovery" test_container_kill_recovery run_test "Journal Recovery Multi-Node" test_journal_recovery_multinode run_test "Journal Recovery Multiple Files" test_journal_recovery_multiple_files diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index f421b0c4..c6b23f27 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -1531,3 +1531,77 @@ func TestStressRandomFailures(t *testing.T) { t.Logf("Stress test completed: %d files created", filesCreated) } + +// TestMultiNodeConcurrentFirstWriteToSameBlock has two nodes write different +// ranges of a new file's first block at the same time. Each node must take the +// inode lock around its read-modify-write of the extents, or the later inode +// write drops the other node's extent and its data. +func TestMultiNodeConcurrentFirstWriteToSameBlock(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 100) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + if err := cluster.WaitAllMounted(15 * time.Second); err != nil { + t.Fatalf("Nodes failed to mount: %v", err) + } + + const files = 20 + name := func(i int) string { return fmt.Sprintf("first_write_%02d.bin", i) } + for i := 0; i < files; i++ { + if err := cluster.Node(0).WriteFile(name(i), nil); err != nil { + t.Fatalf("Failed to create %s: %v", name(i), err) + } + } + // Outlast the kernel's 1s attribute cache so node 1 sees the files. + time.Sleep(1500 * time.Millisecond) + + // Offsets past 64KB keep the files off the SFB path. + const offset0, offset1 = 256 * 1024, 512 * 1024 + data0 := bytes.Repeat([]byte("N0"), 32*1024) + data1 := bytes.Repeat([]byte("N1"), 32*1024) + + for i := 0; i < files; i++ { + errs := cluster.RunConcurrent([]int{0, 1}, func(idx int, node *MountedFS) error { + if idx == 0 { + return node.WriteFileAt(name(i), data0, offset0) + } + return node.WriteFileAt(name(i), data1, offset1) + }) + for idx, err := range errs { + if err != nil { + t.Fatalf("%s: node %d write failed: %v", name(i), idx, err) + } + } + } + + // Read from a freshly mounted node so no cached extents hide the result. + cluster.StopAllNodes() + if err := cluster.StartNode(0, false); err != nil { + t.Fatalf("Failed to remount node 0: %v", err) + } + + lost := 0 + for i := 0; i < files; i++ { + got, err := cluster.Node(0).ReadFile(name(i)) + if err != nil { + t.Fatalf("Failed to read %s: %v", name(i), err) + } + if len(got) < offset1+len(data1) { + t.Errorf("%s: size %d, want at least %d", name(i), len(got), offset1+len(data1)) + lost++ + continue + } + ok0 := bytes.Equal(got[offset0:offset0+len(data0)], data0) + ok1 := bytes.Equal(got[offset1:offset1+len(data1)], data1) + if !ok0 || !ok1 { + t.Errorf("%s: node 0 data intact=%v, node 1 data intact=%v", name(i), ok0, ok1) + lost++ + } + } + if lost > 0 { + t.Fatalf("%d of %d files lost a node's write", lost, files) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 91e92ccc..c8320fc3 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -304,6 +304,15 @@ scenarios: docker: run-tests.sh::test_mixed_size_writes vm: run-vm-tests.sh::test_mixed_size_writes + - id: multinode-concurrent-first-write-same-block + name: Concurrent first write to the same block + description: Two nodes write different ranges of a new file's first block at once; both writes survive + priority: high + environments: + go: multinode_test.go::TestMultiNodeConcurrentFirstWriteToSameBlock + docker: run-tests.sh::test_concurrent_first_write_same_block + vm: run-vm-tests.sh::test_concurrent_first_write_same_block + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets @@ -1420,6 +1429,13 @@ scenarios: environments: go: internal/lock/safety_test.go::TestStaleSharedLockReclaimResetsReaderCount + - id: lock-leaked-shared-count-goes-stale + name: Leaked reader count goes stale + description: New readers do not refresh a shared record's age, so a reader count leaked by a lost update cannot block writers forever + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNewReadersDoNotKeepLeakedSharedLockFresh + - id: lock-region-bounded name: Lock region stays bounded description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index aafbc100..d9018289 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2330,6 +2330,43 @@ test_stability() { # ============================================================================= # Test 10: Two VMs write to same file at different offsets +# Two VMs write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other VM's extent. +test_concurrent_first_write_same_block() { + log_info "Testing concurrent first writes to the same block (2 VMs)..." + ensure_cluster_healthy 2 + + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + ssh_run 1 "touch /mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$size" "$offset0" & + write_pattern_at_on_vm 2 "first_write_$i.bin" "B" "$size" "$offset1" & + wait + done + sleep 2 + + # Each writer caches its own extent, so a lost update shows on either reader. + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at_on_vm 1 "first_write_$i.bin" "B" "$offset1" 4096 || lost=$((lost + 1)) + done + if [[ $lost -gt 0 ]]; then + log_error "$lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log_success "All concurrent first writes survived" + return 0 +} + test_concurrent_same_file_writes() { log_info "Testing concurrent same-file writes (2 VMs)..." ensure_cluster_healthy 2 @@ -2874,6 +2911,7 @@ main() { run_test "Long-Running Stability" test_stability # Run concurrent same-file write tests (using pre-extension pattern) + run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Concurrent Same-File Writes (2 VMs)" test_concurrent_same_file_writes run_test "Concurrent Same-File Writes (3 VMs)" test_concurrent_same_file_writes_3vms run_test "Mixed Size Writes" test_mixed_size_writes From de7683d33d65615bcdde70063953d3ace6e34514 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 09:19:03 -0500 Subject: [PATCH 11/18] test: verify first-write lost updates from a restarted reader The VM runner deletes every file before each test, so new files reuse inode numbers and a node that read earlier files serves stale cached extents. That made the VM check fail even though no write was lost. Co-Authored-By: Claude Opus 5.5 --- tests/docker/scripts/run-tests.sh | 5 ++++- tests/vm/scripts/run-vm-tests.sh | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 4acb3212..0136cc9b 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -1874,7 +1874,10 @@ test_concurrent_first_write_same_block() { write_pattern_at "$writer1" "first_write_$i.bin" B "$size" "$offset1" & wait done - sleep 2 + + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + restart_node "$reader" + wait_node_healthy "$reader" 30 || return 1 local lost=0 for i in $(seq 1 $files); do diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index d9018289..c35f2bd0 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2353,7 +2353,12 @@ test_concurrent_first_write_same_block() { done sleep 2 - # Each writer caches its own extent, so a lost update shows on either reader. + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + stop_vm 1 + start_vm 1 false + wait_vm_healthy 1 || return 1 + setup_goclusterfs 1 false + local lost=0 for i in $(seq 1 $files); do verify_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$offset0" 4096 || lost=$((lost + 1)) From fbb64559919c75c2ee694ad771c762665888132c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 10:49:38 -0500 Subject: [PATCH 12/18] test(lock): age records through a test helper now that writeRecord is gone Co-Authored-By: Claude Opus 5.5 --- internal/lock/safety_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 7c32e451..11a88aa6 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -322,15 +322,7 @@ func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { writer := newTestNode(dev, 0, 1<<20, 3) requireAcquired(t, leaked, LockTypeInode, 2, LockModeShared) - recordID := leaked.hashToRecord(LockTypeInode, 2) - rec, err := leaked.readRecord(recordID) - if err != nil { - t.Fatalf("readRecord: %v", err) - } - rec.AcquireTime = time.Now().Add(-StaleThreshold + 2*time.Second).Unix() - if err := leaked.writeRecord(recordID, rec); err != nil { - t.Fatalf("writeRecord: %v", err) - } + ageRecord(t, leaked, LockTypeInode, 2, StaleThreshold-2*time.Second) requireAcquired(t, reader, LockTypeInode, 2, LockModeShared) if err := reader.Release(LockTypeInode, 2); err != nil { From 6e9e6150a527f016fb72ce2658d277b6d3240d3f Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:53:11 -0500 Subject: [PATCH 13/18] fix(lock): stop same-node writers waiting on their own readers; renew busy records An exclusive request waited for this node's shared holders of the record. inode.Table holds its mutex across Acquire, and a directory listing holds the directory's shared lock while it reads the table, so creating a file while the directory was listed hung the whole mount. Exclusive requests now join the node's existing hold, as they did before this branch. The record on disk stays shared, so other nodes can read but not write meanwhile. Same-node joins never touched disk, so a record kept busy by overlapping keys kept its first AcquireTime and was taken over as stale after 15s while still held. A key joining a hold older than renewAfter now rewrites AcquireTime. Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 2 + internal/inode/lock_test.go | 60 ++++++++++++++++++++++++++ internal/lock/lock.go | 83 ++++++++++++++++++++++++------------ internal/lock/safety_test.go | 51 ++++++++++------------ tests/test-manifest.yaml | 23 ++++++---- 5 files changed, 155 insertions(+), 64 deletions(-) create mode 100644 internal/inode/lock_test.go diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 095e12a0..8a8f188e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -246,6 +246,8 @@ Common causes of a stall: **Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. +**Locks share records.** The lock region has 256 records, and every lock maps to one of them, so unrelated files and the block bitmap can share a record. A node holding one of those locks blocks other nodes from all of them. If two nodes each hold one record and wait for the other's, both wait until one record is 15 seconds old; the takeover then lets both proceed, with the same risk as a stall. This needs nested locks whose records cross, roughly 1 in 65,000 pairs of overlapping operations. + **Safe operating limits:** | Limit | Why | diff --git a/internal/inode/lock_test.go b/internal/inode/lock_test.go new file mode 100644 index 00000000..137a51b1 --- /dev/null +++ b/internal/inode/lock_test.go @@ -0,0 +1,60 @@ +package inode + +import ( + "context" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/lock" +) + +// A directory listing holds the directory's shared lock and then reads the inode +// table, while Write holds the table mutex and asks for the exclusive lock. +func TestWriteDoesNotHangBehindSameNodeReader(t *testing.T) { + dev, cleanup := createTestDevice(t, 16<<20) + defer cleanup() + var uuid [16]byte + copy(uuid[:], "node-a") + lm := lock.NewManager(dev, 8<<20, 1<<20, uuid, 1) + table := NewTable(dev, 0, 4<<20, lm, uuid) + ctx := context.Background() + dir, err := table.Alloc(ctx) + if err != nil { + t.Fatalf("Alloc: %v", err) + } + key := uint64(dir.InodeNum) + + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + t.Fatalf("Acquire shared: %v", err) + } + writeDone := make(chan error, 1) + go func() { + wctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + writeDone <- table.Write(wctx, dir) + }() + time.Sleep(50 * time.Millisecond) + + readDone := make(chan error, 1) + go func() { + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + readDone <- err + return + } + _, err := table.Read(dir.InodeNum) + lm.Release(lock.LockTypeInode, key) + readDone <- err + }() + lm.Release(lock.LockTypeInode, key) + + for name, done := range map[string]chan error{"Write": writeDone, "listing": readDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("%s: %v", name, err) + } + case <-time.After(2 * time.Second): + t.Fatalf("%s did not finish: same-node reader and writer deadlocked", name) + } + } +} diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 270bd483..192fe7c4 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -239,13 +239,14 @@ type heldLock struct { } type heldRecord struct { - mode LockMode - keys int + mode LockMode + keys int + renewed time.Time } -func covers(held, want LockMode) bool { - return held == LockModeExclusive || held == want -} +// renewAfter is the hold age after which a key joining the record rewrites +// AcquireTime, so a record kept busy by overlapping keys never looks stale. +var renewAfter = StaleThreshold / 3 // Errors var ( @@ -330,7 +331,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin gid := getGoroutineID() m.mu.Lock() - if m.reuseHeldKeyLocked(key, mode, gid) { + if m.reuseHeldKeyLocked(key, gid) { m.mu.Unlock() return nil } @@ -395,24 +396,28 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // attemptAcquire makes one acquisition attempt. added reports whether this // call registered the key rather than joining another goroutine's hold. +// +// A key whose record this node already holds joins that hold in any mode. Exclusive +// requests never wait for this node's readers, who may need a mutex the writer holds; +// meanwhile the record on disk stays shared, so other nodes can read but not write. func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { gate := m.recordGate(recordID) gate.Lock() defer gate.Unlock() m.mu.Lock() - if m.reuseHeldKeyLocked(key, mode, gid) { - m.mu.Unlock() - return true, false, nil - } if rec, ok := m.records[recordID]; ok { - if !covers(rec.mode, mode) { - selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + renew := time.Since(rec.renewed) > renewAfter + m.mu.Unlock() + if renew { + m.renewRecord(recordID, rec) + } + m.mu.Lock() + if held, ok := m.heldLocks[key]; ok { + held.refCount++ + held.goroutines[gid]++ m.mu.Unlock() - if selfDeadlock { - return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) - } - return false, false, nil + return true, false, nil } rec.keys++ m.addHeldKeyLocked(key, recordID, mode, gid) @@ -427,15 +432,15 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi } m.mu.Lock() - m.records[recordID] = &heldRecord{mode: mode, keys: 1} + m.records[recordID] = &heldRecord{mode: mode, keys: 1, renewed: time.Now()} m.addHeldKeyLocked(key, recordID, mode, gid) m.mu.Unlock() return true, true, nil } -func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { +func (m *Manager) reuseHeldKeyLocked(key lockKey, gid uint64) bool { held, ok := m.heldLocks[key] - if !ok || !covers(m.records[held.RecordID].mode, mode) { + if !ok || time.Since(m.records[held.RecordID].renewed) > renewAfter { return false } held.refCount++ @@ -452,15 +457,6 @@ func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, } } -func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { - for _, held := range m.heldLocks { - if held.RecordID == recordID && held.goroutines[gid] > 0 { - return true - } - } - return false -} - func (m *Manager) recordGate(recordID uint64) *sync.Mutex { m.mu.Lock() defer m.mu.Unlock() @@ -663,6 +659,37 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } +// renewRecord rewrites AcquireTime on a record this node holds. It runs under +// the record gate. +func (m *Manager) renewRecord(recordID uint64, rec *heldRecord) { + for { + record, err := m.readRecord(recordID) + if err != nil { + log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) + return + } + ours := record.Mode == rec.mode && (rec.mode == LockModeShared || record.OwnerUUID == m.nodeUUID) + if !ours { + log.Printf("[WARN] lock: record %d was taken over while this node held it", recordID) + return + } + renewed := *record + renewed.AcquireTime = time.Now().Unix() + renewed.Sequence++ + ok, err := m.casRecord(recordID, record, &renewed) + if err != nil { + log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) + return + } + if ok { + m.mu.Lock() + rec.renewed = time.Now() + m.mu.Unlock() + return + } + } +} + // releaseRecord releases this node's hold on a lock record. // After successful release, caches the new sequence number to enable // optimistic acquisition on the next lock attempt. diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 11a88aa6..9677f246 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -153,46 +153,41 @@ func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) } -func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { +func TestExclusiveJoinsSameNodeSharedHold(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) requireAcquired(t, a, LockTypeInode, 9, LockModeShared) - - writer := func() error { - errc := make(chan error, 1) - go func() { - err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) - if err == nil { - err = a.Release(LockTypeInode, 9) - } - errc <- err - }() - return <-errc - } - - if err := writer(); !errors.Is(err, ErrLockTimeout) { - t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) - } - if err := a.Release(LockTypeInode, 9); err != nil { - t.Fatalf("Release shared: %v", err) - } - if err := writer(); err != nil { - t.Fatalf("exclusive Acquire after readers left: %v", err) + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + if err := <-errc; err != nil { + t.Fatalf("exclusive Acquire while this node reads the record: %v", err) } } -func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { +func TestJoiningKeyRenewsBusyRecord(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - a := newTestNode(dev, 0, 1<<20, 1) + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + defer func(d time.Duration) { renewAfter = d }(renewAfter) + renewAfter = 0 - requireAcquired(t, a, LockTypeInode, 9, LockModeShared) - err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) - if !errors.Is(err, ErrDeadlockRisk) { - t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 1, pastStale) + requireAcquired(t, a, LockTypeInode, 2, LockModeExclusive) + if err := a.Release(LockTypeInode, 1); err != nil { + t.Fatalf("Release: %v", err) } + + requireBlocked(t, b, LockTypeInode, 3, LockModeExclusive) } func TestStaleLockIsReclaimedFromPausedHolder(t *testing.T) { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index c8320fc3..b9f28881 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1408,19 +1408,26 @@ scenarios: environments: go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases - - id: lock-exclusive-waits-same-node-readers - name: Exclusive waits for same-node readers - description: An exclusive request does not piggyback on a shared hold from the same node + - id: lock-exclusive-joins-same-node-readers + name: Exclusive joins same-node readers + description: An exclusive request on a node that already reads the record joins that hold instead of waiting for its own readers priority: go_only environments: - go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + go: internal/lock/safety_test.go::TestExclusiveJoinsSameNodeSharedHold - - id: lock-upgrade-same-goroutine - name: Shared to exclusive upgrade rejected - description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + - id: lock-busy-record-renewed + name: Busy record renewed on join + description: A key joining a record this node has held past renewAfter rewrites its AcquireTime, so a busy node does not lose the record as stale priority: go_only environments: - go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + go: internal/lock/safety_test.go::TestJoiningKeyRenewsBusyRecord + + - id: inode-write-same-node-reader + name: Inode write does not hang behind same-node reader + description: Table.Write on a directory finishes while another goroutine on the node holds the directory's shared lock and reads the inode table + priority: go_only + environments: + go: internal/inode/lock_test.go::TestWriteDoesNotHangBehindSameNodeReader - id: lock-stale-shared-reclaim name: Stale shared lock reclaim drops dead readers From 4a50602dd767eaa2b6c11e95f59214b889e03a52 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 12:06:25 -0500 Subject: [PATCH 14/18] fix(fs): drop the inode lock before freeing blocks and releasing tree reservations The bitmap lock comes before the inode lock, so frees made while holding it fail the lock order check. Tree reservations released under the inode lock parked their blocks in an in-memory list that unmount drops: appends merging into a full inline extent leaked one block each. writeFailed freed the write's blocks the same way. Both now run after the inode lock is released. writeInternal released the inode lock it took for the size update, then released it again in its deferred cleanup. If another goroutine on the node had joined that lock in between, the second release dropped its hold. All releases now go through one helper that clears lockHeld. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 60 ++++++++++++---------- tests/integration/tree_reservation_test.go | 42 +++++++++++++++ tests/test-manifest.yaml | 11 ++++ tests/vm/scripts/run-vm-tests.sh | 2 +- 4 files changed, 88 insertions(+), 27 deletions(-) create mode 100644 tests/integration/tree_reservation_test.go diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a25fa508..0842a79d 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2100,36 +2100,42 @@ func (f *FS) zeroBlock(physBlock uint64) error { // allocateBlockLocked maps a new data block at blockNum and returns the inode // re-read under the inode lock, which stays held on success. The bitmap comes // before the inode in lock order, so the data block and any extent-tree blocks -// are allocated before the lock is taken. -func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, error) { +// are allocated before the lock is taken. Call the returned release after +// dropping the inode lock. +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, func(), error) { blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) if err != nil { - return nil, nil, syscall.ENOSPC + return nil, nil, nil, syscall.ENOSPC } ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) if err != nil { f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, syscall.ENOSPC + return nil, nil, nil, syscall.ENOSPC } - defer releaseTree() if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, syscall.EIO + releaseTree() + return nil, nil, nil, syscall.EIO } - fail := func(errno error) (*inode.Inode, *extent.Extent, error) { + var fresh *inode.Inode + fail := func(errno error) (*inode.Inode, *extent.Extent, func(), error) { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + if fresh != nil { + f.fs.extentMgr.DiscardPending(ctx, fresh) + } f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) - return nil, nil, errno + releaseTree() + return nil, nil, nil, errno } - fresh, err := f.fs.inodeTbl.Read(f.inodeNum) + fresh, err = f.fs.inodeTbl.Read(f.inodeNum) if err != nil { + fresh = nil return fail(syscall.EIO) } // Insert fails if another node mapped this block, or grew the tree past the reservation, meanwhile. if err := f.fs.extentMgr.Insert(treeCtx, fresh, ext); err != nil { - f.fs.extentMgr.DiscardPending(ctx, fresh) switch { case errors.Is(err, extent.ErrExtentTreeFull): return fail(syscall.EFBIG) @@ -2140,7 +2146,7 @@ func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockN } fresh.AllocatedBlocks += ext.Length fresh.WrittenBlocks += ext.Length - return fresh, ext, nil + return fresh, ext, releaseTree, nil } // writeInternal performs the actual write operation. @@ -2152,9 +2158,18 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i lockHeld := false // ino has changes made under the inode lock that are not on disk yet inoDirty := false - defer func() { + // Tree-block reservations are released after the inode lock, as lock order requires. + var releaseReservations []func() + releaseInodeLock := func() { if lockHeld && f.fs.lockMgr != nil { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + lockHeld = false + } + } + defer func() { + releaseInodeLock() + for _, release := range releaseReservations { + release() } }() @@ -2253,28 +2268,28 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Release inode lock before allocation to respect lock ordering: // bitmap (6) must be acquired before inode (8) - // AllocateBlocks acquires bitmap lock internally if lockHeld && f.fs.lockMgr != nil { // The inode is re-read after allocation, so persist changes made under the lock first. if inoDirty { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { log.Printf("[DEBUG] Write inode %d: persisting inode before allocation failed: %v", f.inodeNum, err) + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) return 0, retryOnTreeConflict(err) } inoDirty = false } - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - lockHeld = false + releaseInodeLock() } if f.fs.lockMgr != nil { - freshIno, newExt, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + freshIno, newExt, releaseTree, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) if allocErr != nil { log.Printf("[DEBUG] Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) return 0, allocErr } lockHeld = true + releaseReservations = append(releaseReservations, releaseTree) ino = freshIno inoDirty = true ext = newExt @@ -2538,9 +2553,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Write inode %d: re-read inode failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, syscall.EIO } // ino was read before the lock, so it can miss another node's extents. @@ -2603,6 +2615,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // On a tree conflict the blocks allocated here are unreferenced. Close the // intent first so journal replay cannot map them again, then free them. writeFailed := func(err error) error { + // The bitmap lock comes before the inode lock, so free only after dropping it. + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) if !errors.Is(err, extent.ErrTreeConflict) { return syscall.EIO @@ -2634,9 +2648,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { log.Printf("[DEBUG] Write inode %d: WriteWithLockHeld failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, writeFailed(err) } } else { @@ -2648,10 +2659,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i inodeWritten = true - // Release the lock we acquired for size update - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } + releaseInodeLock() for _, cow := range cowDone { newCount, decErr := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock) diff --git a/tests/integration/tree_reservation_test.go b/tests/integration/tree_reservation_test.go new file mode 100644 index 00000000..7b5c3787 --- /dev/null +++ b/tests/integration/tree_reservation_test.go @@ -0,0 +1,42 @@ +package integration + +import ( + "syscall" + "testing" +) + +// TestTreeReservationsDoNotLeakBlocks appends to a file whose two inline extents +// are full, so each append reserves an extent-tree block and then merges into the +// last extent without using it. Unused reservations must go back to the bitmap, +// not to an in-memory list that unmount drops, so the whole volume is writable +// again once the file is deleted. +func TestTreeReservationsDoNotLeakBlocks(t *testing.T) { + mfs := NewMountedFS(t, 64) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + // 128KB at block 0 keeps the file off the SFB path; block 2 makes the second extent. + if err := mfs.WriteFileAt("append.bin", chunk(1, 128*1024), 0); err != nil { + t.Fatalf("write at block 0 failed: %v", err) + } + for blk := 2; blk <= 7; blk++ { + if err := mfs.WriteFileAt("append.bin", chunk(byte(blk), 64*1024), int64(blk*mb)); err != nil { + t.Fatalf("write at block %d failed: %v", blk, err) + } + } + remount(t, mfs) + if err := mfs.Remove("append.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + + var st syscall.Statfs_t + if err := syscall.Statfs(mfs.MountPoint, &st); err != nil { + t.Fatalf("Statfs failed: %v", err) + } + // Every data block except the root directory's. + fill := int(st.Blocks-1) * mb + if err := mfs.WriteFile("fill.bin", make([]byte, fill)); err != nil { + t.Fatalf("writing %d MB into an empty volume of %d blocks failed: %v", fill/mb, st.Blocks, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index b9f28881..095828da 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -313,6 +313,17 @@ scenarios: docker: run-tests.sh::test_concurrent_first_write_same_block vm: run-vm-tests.sh::test_concurrent_first_write_same_block + # Docker/VM TODO: single-node behavior that needs a remount and a full-volume + # write; the Go integration test runs the real mount binary already. + - id: tree-reservation-no-leak + name: Extent-tree reservations do not leak blocks + description: Appends that merge into a full inline extent return their unused tree-block reservation, so the whole volume is writable after the file is deleted and the FS remounted + priority: medium + environments: + go: tree_reservation_test.go::TestTreeReservationsDoNotLeakBlocks + docker: TODO + vm: TODO + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index c35f2bd0..4bac69f4 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2329,7 +2329,6 @@ test_stability() { # Concurrent Same-File Write Tests (using pre-extension pattern) # ============================================================================= -# Test 10: Two VMs write to same file at different offsets # Two VMs write different ranges of each new file's first block at once. # Without the inode lock the later inode write drops the other VM's extent. test_concurrent_first_write_same_block() { @@ -2372,6 +2371,7 @@ test_concurrent_first_write_same_block() { return 0 } +# Test 10: Two VMs write to same file at different offsets test_concurrent_same_file_writes() { log_info "Testing concurrent same-file writes (2 VMs)..." ensure_cluster_healthy 2 From fd638fd0ba7abf81e308b8dd58649d2adb11003c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 12:12:45 -0500 Subject: [PATCH 15/18] fix(fs): clear unwritten flags outside the inode lock Since #12, the first write to an unwritten block clears only that block, which can split one extent into three and allocate extent-tree blocks. Under the live inode lock that allocation failed the lock order check, so a write into the middle of a fallocated range returned EIO. The flag is now cleared on an unlocked copy; the write keeps it only if the inode is unchanged once the lock is held, and retries otherwise. The lock stays held from that check through the zero-fill, the data write and the inode commit. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 104 +++++++++++---------- tests/integration/tree_reservation_test.go | 45 +++++++++ tests/test-manifest.yaml | 9 ++ 3 files changed, 109 insertions(+), 49 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 0842a79d..0142defa 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2196,6 +2196,24 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: Read failed: %v", f.inodeNum, err) return 0, syscall.EIO } + // persistAndUnlock writes changes made under the inode lock and drops it, so + // the bitmap lock, which comes first in lock order, can be taken next. + persistAndUnlock := func() error { + if !lockHeld || f.fs.lockMgr == nil { + return nil + } + if inoDirty { + if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { + log.Printf("[DEBUG] Write inode %d: persisting inode before unlocking failed: %v", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + return retryOnTreeConflict(err) + } + inoDirty = false + } + releaseInodeLock() + return nil + } inodeWritten := false defer func() { if !inodeWritten { @@ -2266,20 +2284,9 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } - // Release inode lock before allocation to respect lock ordering: - // bitmap (6) must be acquired before inode (8) - if lockHeld && f.fs.lockMgr != nil { - // The inode is re-read after allocation, so persist changes made under the lock first. - if inoDirty { - if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { - log.Printf("[DEBUG] Write inode %d: persisting inode before allocation failed: %v", f.inodeNum, err) - releaseInodeLock() - f.fs.extentMgr.DiscardPending(ctx, ino) - return 0, retryOnTreeConflict(err) - } - inoDirty = false - } - releaseInodeLock() + // The inode is re-read after allocation, so persist changes made under the lock first. + if err := persistAndUnlock(); err != nil { + return 0, err } if f.fs.lockMgr != nil { @@ -2403,50 +2410,49 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Check if this is an unwritten extent (from fallocate). // If so, we need to clear the Unwritten flag on first write. - if ext.Flags&extent.FlagUnwritten != 0 { - // First write to unwritten block - need to clear flag - if !lockHeld && f.fs.lockMgr != nil { - // Acquire lock before modifying extent - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to acquire lock for unwritten: %v", f.inodeNum, err) - return 0, syscall.EIO - } - lockHeld = true - - // Re-read inode after acquiring lock - freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: re-read for unwritten failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - if freshIno.MetaGeneration != ino.MetaGeneration { - log.Printf("[DEBUG] Write inode %d: MetaGeneration changed during unwritten clear", - f.inodeNum) - return 0, errRetryWrite - } - ino = freshIno - - // Re-lookup extent - another node may have already cleared it - ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: extent disappeared during unwritten clear", - f.inodeNum) + if ext.Flags&extent.FlagUnwritten != 0 && f.fs.lockMgr != nil { + // Splitting the extent may allocate tree blocks, which needs the bitmap lock + // before the inode lock. Clear on an unlocked copy, then keep it only if no + // node wrote the inode before the lock was taken. + if err := persistAndUnlock(); err != nil { + return 0, err + } + cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ctx, ino, blockNum) + if clearErr != nil { + log.Printf("[DEBUG] Write inode %d: clearing Unwritten flag at block %d failed: %v", + f.inodeNum, blockNum, clearErr) + f.fs.extentMgr.DiscardPending(ctx, ino) + if errors.Is(clearErr, extent.ErrStaleTree) { return 0, errRetryWrite } + return 0, syscall.EIO } - - // Clear the Unwritten flag (if still set) + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + log.Printf("[DEBUG] Write inode %d: failed to acquire lock for unwritten: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, syscall.EIO + } + lockHeld = true + freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || freshIno.MetaGeneration != ino.MetaGeneration { + log.Printf("[DEBUG] Write inode %d: inode changed during unwritten clear (err=%v)", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, errRetryWrite + } + inoDirty = inoDirty || cleared + zeroFirst = cleared + if ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum); err != nil { + log.Printf("[DEBUG] Write inode %d: lookup after unwritten clear failed: %v", f.inodeNum, err) + return 0, syscall.EIO + } + } else if ext.Flags&extent.FlagUnwritten != 0 { cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ctx, ino, blockNum) if clearErr != nil { log.Printf("[DEBUG] Write inode %d: clearing Unwritten flag at block %d failed: %v", f.inodeNum, blockNum, clearErr) return 0, syscall.EIO } - if cleared { - inoDirty = true - log.Printf("[DEBUG] Write inode %d: cleared Unwritten flag for block %d", - f.inodeNum, blockNum) - } zeroFirst = cleared } diff --git a/tests/integration/tree_reservation_test.go b/tests/integration/tree_reservation_test.go index 7b5c3787..5e8a3498 100644 --- a/tests/integration/tree_reservation_test.go +++ b/tests/integration/tree_reservation_test.go @@ -40,3 +40,48 @@ func TestTreeReservationsDoNotLeakBlocks(t *testing.T) { t.Fatalf("writing %d MB into an empty volume of %d blocks failed: %v", fill/mb, st.Blocks, err) } } + +// TestWriteInsideFallocatedRange writes 4KB into the middle of a fallocated +// range. Clearing the unwritten flag splits one extent into three, which needs +// extent-tree blocks, and the rest of the block must read as zeros even though +// the block held another file's data before. +func TestWriteInsideFallocatedRange(t *testing.T) { + mfs := NewMountedFS(t, 64) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + old := make([]byte, 4*mb) + for i := range old { + old[i] = 0xEE + } + if err := mfs.WriteFile("old.bin", old); err != nil { + t.Fatalf("write old.bin failed: %v", err) + } + if err := mfs.Remove("old.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + + if err := mfs.WriteFile("falloc.bin", nil); err != nil { + t.Fatalf("create falloc.bin failed: %v", err) + } + f, err := mfs.OpenFile("falloc.bin") + if err != nil { + t.Fatalf("OpenFile failed: %v", err) + } + if err := syscall.Fallocate(int(f.Fd()), 0, 0, 4*mb); err != nil { + f.Close() + t.Fatalf("Fallocate failed: %v", err) + } + data := chunk(7, 4096) + const at = mb + 512*1024 + if _, err := f.WriteAt(data, at); err != nil { + f.Close() + t.Fatalf("write inside the fallocated range failed: %v", err) + } + f.Close() + + remount(t, mfs) + want := make([]byte, 4*mb) + copy(want[at:], data) + checkFile(t, mfs, "falloc.bin", want) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 095828da..14ceda46 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -324,6 +324,15 @@ scenarios: docker: TODO vm: TODO + - id: write-inside-fallocated-range + name: Write inside a fallocated range + description: A 4KB write into the middle of a fallocated range splits the unwritten extent under the live lock and zero-fills the rest of the block + priority: medium + environments: + go: tree_reservation_test.go::TestWriteInsideFallocatedRange + docker: TODO + vm: TODO + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets From 775007597a103be37b0677dd092c4479120f7ad8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:19:15 -0500 Subject: [PATCH 16/18] fix(lock): wait for same-node readers again; never hold a local mutex across Acquire Letting an exclusive request join this node's shared hold kept the record shared on disk, so two nodes that both read a directory could each take it exclusively and lose each other's entries. Exclusive requests wait for this node's readers again. The hang that change avoided came from local mutexes held across Acquire: a writer held inode.Table's mutex while waiting for a reader that needed it. inode.Table, bitmap, refcount and the snapshot registry now take the distributed lock before their mutex. Renewal now covers exclusive holds only, so a dead node's reader count no longer stays fresh. A join that finds its record taken over fails with ErrLockLost, and later joins to that hold fail too. Pending tree changes are discarded after the inode lock is dropped on every error path. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 12 ++--- internal/goclusterfs/fs.go | 2 + internal/inode/inode.go | 53 +++++++-------------- internal/lock/lock.go | 76 ++++++++++++++++++++++--------- internal/lock/safety_test.go | 86 +++++++++++++++++++++++++++++++---- internal/refcount/refcount.go | 42 ++++++++--------- internal/snapshot/snapshot.go | 30 ++++++------ tests/test-manifest.yaml | 31 +++++++++++-- 8 files changed, 218 insertions(+), 114 deletions(-) diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 303173a0..2ee0a5d9 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -89,9 +89,6 @@ func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) { return nil, nil } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { @@ -100,6 +97,9 @@ func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) { defer r.lockMgr.Release(lock.LockTypeBlockBitmap, 0) } + r.mu.Lock() + defer r.mu.Unlock() + allocated := make([]uint64, 0, count) // Scan through sectors looking for free blocks @@ -170,9 +170,6 @@ func (r *Region) Free(ctx context.Context, blockNum uint64) error { return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { @@ -181,6 +178,9 @@ func (r *Region) Free(ctx context.Context, blockNum uint64) error { defer r.lockMgr.Release(lock.LockTypeBlockBitmap, 0) } + r.mu.Lock() + defer r.mu.Unlock() + sectorIdx := blockNum / BitsPerSector bitOffset := blockNum % BitsPerSector byteIdx := bitOffset / 8 diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 0142defa..384673f6 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2217,6 +2217,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i inodeWritten := false defer func() { if !inodeWritten { + // Discarding frees tree blocks, which needs the bitmap lock: drop the inode lock first. + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) } }() diff --git a/internal/inode/inode.go b/internal/inode/inode.go index edc3f4f4..8daf0578 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -202,9 +202,6 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { return nil, ErrInodeOutOfRange } - t.mu.Lock() - defer t.mu.Unlock() - // Acquire distributed lock if available if t.lockMgr != nil { if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inode.InodeNum), lock.LockModeExclusive); err != nil { @@ -213,6 +210,9 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { defer t.lockMgr.Release(lock.LockTypeInode, uint64(inode.InodeNum)) } + t.mu.Lock() + defer t.mu.Unlock() + return t.writeWithTree(inode) } @@ -311,11 +311,10 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { return nil, ErrInodeOutOfRange } - t.mu.Lock() - defer t.mu.Unlock() - // Re-read to verify slot is still free (another node may have claimed it) + t.mu.RLock() existing, err := t.readInodeUnlocked(num) + t.mu.RUnlock() if err != nil { return nil, fmt.Errorf("failed to read inode slot: %w", err) } @@ -331,6 +330,9 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { defer t.lockMgr.Release(lock.LockTypeInode, uint64(num)) } + t.mu.Lock() + defer t.mu.Unlock() + // Double-check after acquiring lock (another node could have won the race) existing, err = t.readInodeUnlocked(num) if err != nil { @@ -367,37 +369,16 @@ func (t *Table) newInodeInSlot(num uint32, free *Inode) *Inode { // Deprecated: Use FindFreeSlot() + ClaimSlot() for operations that need to allocate // blocks before claiming the inode (to respect lock ordering: BlockBitmap before Inode). func (t *Table) Alloc(ctx context.Context) (*Inode, error) { - t.mu.Lock() - defer t.mu.Unlock() - - // Scan for a free inode starting from InodeFirstUsable - for num := InodeFirstUsable; uint64(num) < t.totalInodes; num++ { - inode, err := t.readInodeUnlocked(num) + for { + num, err := t.FindFreeSlot() if err != nil { - continue // Skip unreadable inodes + return nil, err } - - // Check if inode is free (Magic == 0) - if inode.Magic == 0 { - newInode := t.newInodeInSlot(num, inode) - - // Acquire distributed lock if available - if t.lockMgr != nil { - if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(num), lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire inode lock: %w", err) - } - defer t.lockMgr.Release(lock.LockTypeInode, uint64(num)) - } - - if err := t.writeInodeUnlocked(newInode); err != nil { - return nil, fmt.Errorf("failed to write new inode: %w", err) - } - - return newInode, nil + ino, err := t.ClaimSlot(ctx, num) + if !errors.Is(err, ErrInodeInUse) { + return ino, err } } - - return nil, ErrInodeFull } // Free marks an inode as free. @@ -410,9 +391,6 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error { return fmt.Errorf("cannot free reserved inode %d", inodeNum) } - t.mu.Lock() - defer t.mu.Unlock() - // Acquire distributed lock if available if t.lockMgr != nil { if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inodeNum), lock.LockModeExclusive); err != nil { @@ -421,6 +399,9 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error { defer t.lockMgr.Release(lock.LockTypeInode, uint64(inodeNum)) } + t.mu.Lock() + defer t.mu.Unlock() + // Read current inode inode, err := t.readInodeUnlocked(inodeNum) if err != nil { diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 192fe7c4..6452f652 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -242,10 +242,15 @@ type heldRecord struct { mode LockMode keys int renewed time.Time + lost bool } -// renewAfter is the hold age after which a key joining the record rewrites -// AcquireTime, so a record kept busy by overlapping keys never looks stale. +func covers(held, want LockMode) bool { + return held == LockModeExclusive || held == want +} + +// renewAfter is the age of an exclusive hold after which a key joining the record +// rewrites AcquireTime, so a record kept busy by overlapping keys never looks stale. var renewAfter = StaleThreshold / 3 // Errors @@ -255,6 +260,7 @@ var ( ErrNotLockOwner = errors.New("not the lock owner") ErrDeadlockRisk = errors.New("lock order violation would cause deadlock") ErrInvalidRecord = errors.New("invalid lock record") + ErrLockLost = errors.New("lock record was taken over by another node") ) // NewManager creates a new lock manager. @@ -331,7 +337,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin gid := getGoroutineID() m.mu.Lock() - if m.reuseHeldKeyLocked(key, gid) { + if m.reuseHeldKeyLocked(key, mode, gid) { m.mu.Unlock() return nil } @@ -397,9 +403,9 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // attemptAcquire makes one acquisition attempt. added reports whether this // call registered the key rather than joining another goroutine's hold. // -// A key whose record this node already holds joins that hold in any mode. Exclusive -// requests never wait for this node's readers, who may need a mutex the writer holds; -// meanwhile the record on disk stays shared, so other nodes can read but not write. +// A key joins this node's hold on its record when the hold's mode covers the +// request. An exclusive request waits for this node's own readers, so callers +// must not hold a local mutex across Acquire that those readers may need. func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { gate := m.recordGate(recordID) gate.Lock() @@ -407,10 +413,24 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi m.mu.Lock() if rec, ok := m.records[recordID]; ok { - renew := time.Since(rec.renewed) > renewAfter + switch { + case rec.lost: + m.mu.Unlock() + return false, false, ErrLockLost + case !covers(rec.mode, mode): + selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + m.mu.Unlock() + if selfDeadlock { + return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) + } + return false, false, nil + } + renew := rec.mode == LockModeExclusive && time.Since(rec.renewed) > renewAfter m.mu.Unlock() if renew { - m.renewRecord(recordID, rec) + if err := m.renewRecord(recordID, rec); err != nil { + return false, false, err + } } m.mu.Lock() if held, ok := m.heldLocks[key]; ok { @@ -438,9 +458,13 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi return true, true, nil } -func (m *Manager) reuseHeldKeyLocked(key lockKey, gid uint64) bool { +func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { held, ok := m.heldLocks[key] - if !ok || time.Since(m.records[held.RecordID].renewed) > renewAfter { + if !ok { + return false + } + rec := m.records[held.RecordID] + if rec.lost || !covers(rec.mode, mode) || (rec.mode == LockModeExclusive && time.Since(rec.renewed) > renewAfter) { return false } held.refCount++ @@ -457,6 +481,15 @@ func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, } } +func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { + for _, held := range m.heldLocks { + if held.RecordID == recordID && held.goroutines[gid] > 0 { + return true + } + } + return false +} + func (m *Manager) recordGate(recordID uint64) *sync.Mutex { m.mu.Lock() defer m.mu.Unlock() @@ -659,33 +692,34 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } -// renewRecord rewrites AcquireTime on a record this node holds. It runs under -// the record gate. -func (m *Manager) renewRecord(recordID uint64, rec *heldRecord) { +// renewRecord rewrites AcquireTime on a record this node holds exclusively. It +// runs under the record gate. If another node took the record over as stale, +// the hold is marked lost and no new key may join it. +func (m *Manager) renewRecord(recordID uint64, rec *heldRecord) error { for { record, err := m.readRecord(recordID) if err != nil { - log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) - return + return err } - ours := record.Mode == rec.mode && (rec.mode == LockModeShared || record.OwnerUUID == m.nodeUUID) - if !ours { + if record.Mode != LockModeExclusive || record.OwnerUUID != m.nodeUUID { log.Printf("[WARN] lock: record %d was taken over while this node held it", recordID) - return + m.mu.Lock() + rec.lost = true + m.mu.Unlock() + return ErrLockLost } renewed := *record renewed.AcquireTime = time.Now().Unix() renewed.Sequence++ ok, err := m.casRecord(recordID, record, &renewed) if err != nil { - log.Printf("[WARN] lock: renewing record %d: %v", recordID, err) - return + return err } if ok { m.mu.Lock() rec.renewed = time.Now() m.mu.Unlock() - return + return nil } } } diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 9677f246..71063153 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -153,22 +153,71 @@ func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) } -func TestExclusiveJoinsSameNodeSharedHold(t *testing.T) { +func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) requireAcquired(t, a, LockTypeInode, 9, LockModeShared) - errc := make(chan error, 1) - go func() { - err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) - if err == nil { - err = a.Release(LockTypeInode, 9) + + writer := func() error { + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + return <-errc + } + + if err := writer(); !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) + } + if err := a.Release(LockTypeInode, 9); err != nil { + t.Fatalf("Release shared: %v", err) + } + if err := writer(); err != nil { + t.Fatalf("exclusive Acquire after readers left: %v", err) + } +} + +func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) + if !errors.Is(err, ErrDeadlockRisk) { + t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + } +} + +func TestReadersOnTwoNodesCannotBothWrite(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + requireAcquired(t, a, LockTypeInode, 77, LockModeShared) + requireAcquired(t, b, LockTypeInode, 77, LockModeShared) + + errs := make(chan error, 2) + for _, m := range []*Manager{a, b} { + go func(m *Manager) { + err := acquireWithin(m, 50*time.Millisecond, LockTypeInode, 77, LockModeExclusive) + if err == nil { + m.Release(LockTypeInode, 77) + } + errs <- err + }(m) + } + for i := 0; i < 2; i++ { + if err := <-errs; !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while both nodes read: got %v, want ErrLockTimeout", err) } - errc <- err - }() - if err := <-errc; err != nil { - t.Fatalf("exclusive Acquire while this node reads the record: %v", err) } } @@ -327,3 +376,20 @@ func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { requireAcquired(t, writer, LockTypeInode, 2, LockModeExclusive) } + +func TestJoinFailsAfterRecordTakenOver(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + defer func(d time.Duration) { renewAfter = d }(renewAfter) + renewAfter = 0 + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 1, pastStale) + requireAcquired(t, b, LockTypeInode, 2, LockModeExclusive) + + if err := acquireWithin(a, time.Second, LockTypeInode, 3, LockModeExclusive); !errors.Is(err, ErrLockLost) { + t.Fatalf("joining a record another node took over: got %v, want ErrLockLost", err) + } +} diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 7c221f91..6863179c 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -130,9 +130,6 @@ func (r *Region) Get(ctx context.Context, blockNum uint64) (Entry, error) { return Entry{}, ErrBlockOutOfRange } - r.mu.RLock() - defer r.mu.RUnlock() - // Acquire distributed lock if available (shared mode for read) if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeShared); err != nil { @@ -141,6 +138,9 @@ func (r *Region) Get(ctx context.Context, blockNum uint64) (Entry, error) { defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.RLock() + defer r.mu.RUnlock() + return r.readEntry(blockNum) } @@ -151,9 +151,6 @@ func (r *Region) Increment(ctx context.Context, blockNum uint64) (uint32, error) return 0, ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -162,6 +159,9 @@ func (r *Region) Increment(ctx context.Context, blockNum uint64) (uint32, error) defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return 0, err @@ -188,9 +188,6 @@ func (r *Region) Decrement(ctx context.Context, blockNum uint64) (uint32, error) return 0, ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -199,6 +196,9 @@ func (r *Region) Decrement(ctx context.Context, blockNum uint64) (uint32, error) defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return 0, err @@ -224,9 +224,6 @@ func (r *Region) Set(ctx context.Context, blockNum uint64, count uint32, flags E return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -235,6 +232,9 @@ func (r *Region) Set(ctx context.Context, blockNum uint64, count uint32, flags E defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry := Entry{ RefCount: count, Flags: flags, @@ -248,9 +248,6 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -259,6 +256,9 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return err @@ -282,9 +282,6 @@ func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint } } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -293,6 +290,9 @@ func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + results := make([]uint32, len(blockNums)) // Group blocks by sector for efficient I/O @@ -344,9 +344,6 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint } } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -355,6 +352,9 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + results := make([]uint32, len(blockNums)) // Group blocks by sector for efficient I/O diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 0de4079d..ddc0f9a8 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -261,9 +261,6 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, return nil, ErrNameTooLong } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { // Use lock type for snapshot registry (we'll use a constant ID since registry is single) @@ -273,6 +270,9 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + // Find a free slot slot, err := r.findFreeSlot() if err != nil { @@ -420,9 +420,6 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -431,6 +428,9 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { @@ -451,9 +451,6 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -462,6 +459,9 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + // Find and update the descriptor for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) @@ -486,9 +486,6 @@ func (r *Registry) SetRefCountJobID(ctx context.Context, snapshotID, jobID uint6 return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -497,6 +494,9 @@ func (r *Registry) SetRefCountJobID(ctx context.Context, snapshotID, jobID uint6 defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { @@ -524,9 +524,6 @@ func (r *Registry) Remove(ctx context.Context, snapshotID uint64) error { return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -535,6 +532,9 @@ func (r *Registry) Remove(ctx context.Context, snapshotID uint64) error { defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 14ceda46..37e2ac74 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1428,16 +1428,37 @@ scenarios: environments: go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases - - id: lock-exclusive-joins-same-node-readers - name: Exclusive joins same-node readers - description: An exclusive request on a node that already reads the record joins that hold instead of waiting for its own readers + - id: lock-exclusive-waits-same-node-readers + name: Exclusive waits for same-node readers + description: An exclusive request does not join a shared hold from the same node; it waits until this node's readers release priority: go_only environments: - go: internal/lock/safety_test.go::TestExclusiveJoinsSameNodeSharedHold + go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + + - id: lock-upgrade-same-goroutine + name: Shared to exclusive upgrade rejected + description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + + - id: lock-two-node-readers-cannot-write + name: Readers on two nodes cannot both write + description: When two nodes hold a record shared, neither node's exclusive request succeeds + priority: go_only + environments: + go: internal/lock/safety_test.go::TestReadersOnTwoNodesCannotBothWrite + + - id: lock-join-fails-after-takeover + name: Join fails after takeover + description: A key joining an exclusive hold that another node took over as stale gets ErrLockLost instead of joining + priority: go_only + environments: + go: internal/lock/safety_test.go::TestJoinFailsAfterRecordTakenOver - id: lock-busy-record-renewed name: Busy record renewed on join - description: A key joining a record this node has held past renewAfter rewrites its AcquireTime, so a busy node does not lose the record as stale + description: A key joining an exclusive record this node has held past renewAfter rewrites its AcquireTime, so a busy node does not lose the record as stale priority: go_only environments: go: internal/lock/safety_test.go::TestJoiningKeyRenewsBusyRecord From e8f5aacc9505e9f7042ee72ef434831dece674b2 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:23:36 -0500 Subject: [PATCH 17/18] docs(operations): note that read locks are not renewed Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 8a8f188e..301a863a 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -246,6 +246,8 @@ Common causes of a stall: **Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. +**Read locks are not renewed.** A node whose overlapping lookups keep a directory read-locked for more than 15 seconds can lose that lock to another node's writer. Its readers may then see a half-written directory. Two writers still cannot overlap. + **Locks share records.** The lock region has 256 records, and every lock maps to one of them, so unrelated files and the block bitmap can share a record. A node holding one of those locks blocks other nodes from all of them. If two nodes each hold one record and wait for the other's, both wait until one record is 15 seconds old; the takeover then lets both proceed, with the same risk as a stall. This needs nested locks whose records cross, roughly 1 in 65,000 pairs of overlapping operations. **Safe operating limits:** From 81279c62218743dbff2bea190da98e64decd65ee Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:18:53 -0500 Subject: [PATCH 18/18] fix(fs): keep blocks a mid-write persist committed when the final inode write fails Before allocating its next block, a write persists the inode, which maps the blocks allocated so far. If the final inode write then failed, writeFailed freed every block of the write, including the ones the inode on disk still mapped, so the retry handed them out again. It now frees only allocations and copy-on-write copies made after the last persist, and drops the refcounts of copies that persist already committed. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 29 ++++++- internal/goclusterfs/lock_commit_test.go | 99 ++++++++++++++++++++++++ tests/test-manifest.yaml | 7 ++ 3 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 internal/goclusterfs/lock_commit_test.go diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 384673f6..c1309e31 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2097,6 +2097,10 @@ func (f *FS) zeroBlock(physBlock uint64) error { return err } +// testHookBeforeInodeCommit, when set by a test, runs before writeInternal's final +// inode write; an error it returns is handled as if that write had failed. +var testHookBeforeInodeCommit func() error + // allocateBlockLocked maps a new data block at blockNum and returns the inode // re-read under the inode lock, which stays held on success. The bitmap comes // before the inode in lock order, so the data block and any extent-tree blocks @@ -2182,6 +2186,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i }, 0) // Snapshot refcounts drop only after the inode write commits the copies. var cowDone []*extent.CoWResult + // How many of allocatedExtents and cowDone a mid-write persist already put on disk. + committedAllocs, committedCoW := 0, 0 // Abort transaction on error path defer func() { if txn != nil && f.fs.journal != nil { @@ -2210,6 +2216,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return retryOnTreeConflict(err) } inoDirty = false + committedAllocs, committedCoW = len(allocatedExtents), len(cowDone) } releaseInodeLock() return nil @@ -2620,8 +2627,9 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i ino.Size = newSize ino.Mtime = newMtime - // On a tree conflict the blocks allocated here are unreferenced. Close the - // intent first so journal replay cannot map them again, then free them. + // On a tree conflict the blocks allocated since the last mid-write persist are + // unreferenced. Close the intent first so journal replay cannot map them again, + // then free them. Blocks the persist committed stay mapped on disk. writeFailed := func(err error) error { // The bitmap lock comes before the inode lock, so free only after dropping it. releaseInodeLock() @@ -2636,12 +2644,12 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } var unused []uint64 - for _, alloc := range allocatedExtents { + for _, alloc := range allocatedExtents[committedAllocs:] { for i := uint64(0); i < uint64(alloc.count); i++ { unused = append(unused, alloc.physicalBlock+i) } } - for _, cow := range cowDone { + for _, cow := range cowDone[committedCoW:] { unused = append(unused, cow.NewPhysBlock) } for _, b := range unused { @@ -2649,9 +2657,22 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: freeing block %d after tree conflict failed: %v", f.inodeNum, b, err) } } + // The retry finds these blocks already copied, so release the old ones now. + for _, cow := range cowDone[:committedCoW] { + if _, err := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock); err != nil { + log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", + f.inodeNum, cow.OldPhysBlock, err) + } + } return errRetryWrite } + if testHookBeforeInodeCommit != nil { + if err := testHookBeforeInodeCommit(); err != nil { + return 0, writeFailed(err) + } + } + // Write inode - use WriteWithLockHeld if we hold the lock if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { diff --git a/internal/goclusterfs/lock_commit_test.go b/internal/goclusterfs/lock_commit_test.go new file mode 100644 index 00000000..bbd41b6d --- /dev/null +++ b/internal/goclusterfs/lock_commit_test.go @@ -0,0 +1,99 @@ +package goclusterfs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/supporttools/GoClusterFS/internal/bitmap" + "github.com/supporttools/GoClusterFS/internal/dir" + "github.com/supporttools/GoClusterFS/internal/extent" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +// openLockedTreeFS is openTreeFS with a lock manager, as on a real mount. +func openLockedTreeFS(t *testing.T, path string) *FS { + t.Helper() + dev, err := storage.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { dev.Close() }) + lockPath := filepath.Join(t.TempDir(), "locks.img") + if err := os.WriteFile(lockPath, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(lockPath, 1<<20); err != nil { + t.Fatal(err) + } + lockDev, err := storage.Open(lockPath) + if err != nil { + t.Fatalf("open %s: %v", lockPath, err) + } + t.Cleanup(func() { lockDev.Close() }) + + var nodeUUID [16]byte + copy(nodeUUID[:], "testnode12345678") + lm := lock.NewManager(lockDev, 0, 1<<20, nodeUUID, 1) + bm := bitmap.NewRegion(dev, treeFSBitmapOffset, treeFSDataBlocks, lm, nodeUUID) + inodeTbl := inode.NewTable(dev, treeFSInodeOffset, 4096*inode.InodeSize, lm, nodeUUID) + extentMgr := extent.NewManager(dev, bm, lm, treeFSDataOffset) + inodeTbl.SetTreeFlusher(extentMgr) + return &FS{ + dev: dev, + lockMgr: lm, + bitmapMgr: bm, + inodeTbl: inodeTbl, + extentMgr: extentMgr, + dirMgr: dir.NewManager(dev, inodeTbl, extentMgr, lm, treeFSDataOffset), + nodeUUID: nodeUUID, + dataOffset: treeFSDataOffset, + } +} + +// A write over two new blocks commits the first one to the on-disk inode before +// it allocates the second. If the final inode write then fails, the retry must +// find the first block still allocated, not handed out again as the second. +func TestFailedWriteKeepsBlocksCommittedMidWrite(t *testing.T) { + fsys := openLockedTreeFS(t, newTreeFSImage(t)) + f := newRegularFile(t, fsys) + + conflicts := 1 + testHookBeforeInodeCommit = func() error { + if conflicts > 0 { + conflicts-- + return extent.ErrTreeConflict + } + return nil + } + defer func() { testHookBeforeInodeCommit = nil }() + + data := pattern(3, BlockSize+4096) + writeAt(t, f, data, 0) + if conflicts != 0 { + t.Fatal("the final inode write never ran") + } + + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatalf("read inode: %v", err) + } + seen := map[uint64]bool{} + for blk := uint64(0); blk < 2; blk++ { + ext, off, err := fsys.extentMgr.Lookup(ino, blk) + if err != nil { + t.Fatalf("lookup block %d: %v", blk, err) + } + phys := ext.PhysicalBlock + off + if seen[phys] { + t.Fatalf("logical blocks 0 and 1 both map to physical block %d", phys) + } + seen[phys] = true + if free, err := fsys.bitmapMgr.IsFree(phys); err != nil || free { + t.Fatalf("physical block %d is mapped but free in the bitmap (err=%v)", phys, err) + } + } + checkAt(t, f, data, 0) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 368dd63c..1dc2e029 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1539,6 +1539,13 @@ scenarios: environments: go: internal/lock/safety_test.go::TestNewReadersDoNotKeepLeakedSharedLockFresh + - id: write-failure-keeps-committed-blocks + name: Failed write keeps blocks committed mid-write + description: When the final inode write fails after a mid-write persist committed a new block, the retry finds that block still allocated and mapped + priority: go_only + environments: + go: internal/goclusterfs/lock_commit_test.go::TestFailedWriteKeepsBlocksCommittedMidWrite + - id: lock-region-bounded name: Lock region stays bounded description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it