diff --git a/cmd/mount/main.go b/cmd/mount/main.go index cfec2e25..e77fd5d9 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -274,7 +274,7 @@ func main() { } // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency - gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, nodeUUID, dataOffset) + gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) // Set up cross-node recovery handler for failed node journal replay recoveryCtx := &journal.RecoveryContext{ diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index ec6fe3d2..de88a87b 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -241,12 +241,46 @@ rm /mnt/fs/.snapshots/snapshot_name iscsiadm -m node -T -p --logout ``` +### Distributed Lock Limits + +Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, SFB containers, snapshots) through lock records on the shared device. Read this before you run more than one node. + +**How a lock goes stale.** A lock record stores the wall-clock time it was taken. Nothing renews it. Any other node may take over a lock that is older than 15 seconds (`StaleThreshold`), judged by that node's own clock. This is how locks held by a crashed node get freed. + +**The hazard: a stalled node keeps writing.** If a node stops for more than 15 seconds while it holds a lock, another node takes the lock. When the stalled node resumes, it finishes its writes. No write path checks that the lock is still held, and there is no fencing token. The node only learns it lost the lock when it releases it (`ErrNotLockOwner`), and callers ignore that error. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. + +Common causes of a stall: + +- VM pause, suspend, live migration, or a memory snapshot +- `docker pause`, cgroup freezer, `SIGSTOP`, or a debugger on the mount process +- Heavy swapping or a long GC pause +- Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover + +**Lock takeover is not atomic.** Taking a lock is a read followed by a write, not an atomic compare-and-swap. Two nodes that read a free record within one disk round trip of each other can both take it. This is rare but more likely on busy locks such as the block bitmap. The same race can lose an update to a read lock's reader count. The lock then looks held until it is 15 seconds old, so writers to that directory or file can stall for up to 15 seconds. + +**Read locks are not renewed.** A node whose overlapping lookups keep a directory read-locked for more than 15 seconds can lose that lock to another node's writer. Its readers may then see a half-written directory. Two writers still cannot overlap. + +**Locks share records.** The lock region has 256 records, and every lock maps to one of them, so unrelated files and the block bitmap can share a record. A node holding one of those locks blocks other nodes from all of them. If two nodes each hold one record and wait for the other's, both wait until one record is 15 seconds old; the takeover then lets both proceed, with the same risk as a stall. This needs nested locks whose records cross, roughly 1 in 65,000 pairs of overlapping operations. + +**Safe operating limits:** + +| Limit | Why | +|-------|-----| +| Keep node clocks within 1 second (chrony or NTP) | Clock skew S shortens the stale threshold to 15s − S | +| No node may stall for 10 seconds or more while mounted | Leaves margin below the 15s threshold | +| Storage I/O must fail, not queue, within 10 seconds | A queued write that lands after takeover corrupts metadata. Do not use multipath `queue_if_no_path`; lower iSCSI `node.session.timeo.replacement_timeout` (default 120s) | +| Unmount before you pause, migrate or snapshot a node VM | Unmounting releases its locks | + +If a node did stall for more than 15 seconds while mounted (hung-task warnings in `dmesg`, a VM pause, a path failover), stop its mount process before it does more writes, then check the data from a healthy node. + +What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks 15 seconds after it acquired them. + ### Handling Node Failure When a node fails unexpectedly: 1. **Other nodes continue operating** - Distributed locking handles failed node -2. **Lock cleanup** - Stale locks are detected via heartbeat timeout +2. **Lock cleanup** - Other nodes take over the failed node's locks once they are 15 seconds old (lock age, not node heartbeat) 3. **No immediate action required** - System is self-healing **When failed node recovers**: diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 303173a0..2ee0a5d9 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -89,9 +89,6 @@ func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) { return nil, nil } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { @@ -100,6 +97,9 @@ func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) { defer r.lockMgr.Release(lock.LockTypeBlockBitmap, 0) } + r.mu.Lock() + defer r.mu.Unlock() + allocated := make([]uint64, 0, count) // Scan through sectors looking for free blocks @@ -170,9 +170,6 @@ func (r *Region) Free(ctx context.Context, blockNum uint64) error { return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { @@ -181,6 +178,9 @@ func (r *Region) Free(ctx context.Context, blockNum uint64) error { defer r.lockMgr.Release(lock.LockTypeBlockBitmap, 0) } + r.mu.Lock() + defer r.mu.Unlock() + sectorIdx := blockNum / BitsPerSector bitOffset := blockNum % BitsPerSector byteIdx := bitOffset / 8 diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index fac3e201..c1309e31 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -194,11 +194,13 @@ func NewWithComponents( tm *trim.Manager, // May be nil if TRIM is disabled sfbMgr *sfb.Manager, // May be nil if SFB is disabled refcountReg *refcount.Region, // May be nil if snapshots not enabled + lockMgr *lock.Manager, // May be nil for single-node use nodeUUID [16]byte, dataOffset uint64, ) *FS { return &FS{ dev: dev, + lockMgr: lockMgr, bitmapMgr: bm, inodeTbl: inodeTbl, extentMgr: extentMgr, @@ -2095,6 +2097,62 @@ func (f *FS) zeroBlock(physBlock uint64) error { return err } +// testHookBeforeInodeCommit, when set by a test, runs before writeInternal's final +// inode write; an error it returns is handled as if that write had failed. +var testHookBeforeInodeCommit func() error + +// allocateBlockLocked maps a new data block at blockNum and returns the inode +// re-read under the inode lock, which stays held on success. The bitmap comes +// before the inode in lock order, so the data block and any extent-tree blocks +// are allocated before the lock is taken. Call the returned release after +// dropping the inode lock. +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, func(), error) { + blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) + if err != nil { + return nil, nil, nil, syscall.ENOSPC + } + ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} + treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) + if err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + return nil, nil, nil, syscall.ENOSPC + } + + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + releaseTree() + return nil, nil, nil, syscall.EIO + } + var fresh *inode.Inode + fail := func(errno error) (*inode.Inode, *extent.Extent, func(), error) { + f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + if fresh != nil { + f.fs.extentMgr.DiscardPending(ctx, fresh) + } + f.fs.bitmapMgr.Free(ctx, ext.PhysicalBlock) + releaseTree() + return nil, nil, nil, errno + } + fresh, err = f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + fresh = nil + return fail(syscall.EIO) + } + // Insert fails if another node mapped this block, or grew the tree past the reservation, meanwhile. + if err := f.fs.extentMgr.Insert(treeCtx, fresh, ext); err != nil { + switch { + case errors.Is(err, extent.ErrExtentTreeFull): + return fail(syscall.EFBIG) + case errors.Is(err, extent.ErrCorruptTree): + return fail(syscall.EIO) + } + return fail(errRetryWrite) + } + fresh.AllocatedBlocks += ext.Length + fresh.WrittenBlocks += ext.Length + return fresh, ext, releaseTree, nil +} + // writeInternal performs the actual write operation. // Returns errRetryWrite if a concurrent modification is detected and the caller should retry. func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (int, error) { @@ -2102,9 +2160,20 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Track if we hold the inode lock lockHeld := false - defer func() { + // ino has changes made under the inode lock that are not on disk yet + inoDirty := false + // Tree-block reservations are released after the inode lock, as lock order requires. + var releaseReservations []func() + releaseInodeLock := func() { if lockHeld && f.fs.lockMgr != nil { f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) + lockHeld = false + } + } + defer func() { + releaseInodeLock() + for _, release := range releaseReservations { + release() } }() @@ -2117,6 +2186,8 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i }, 0) // Snapshot refcounts drop only after the inode write commits the copies. var cowDone []*extent.CoWResult + // How many of allocatedExtents and cowDone a mid-write persist already put on disk. + committedAllocs, committedCoW := 0, 0 // Abort transaction on error path defer func() { if txn != nil && f.fs.journal != nil { @@ -2131,9 +2202,30 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: Read failed: %v", f.inodeNum, err) return 0, syscall.EIO } + // persistAndUnlock writes changes made under the inode lock and drops it, so + // the bitmap lock, which comes first in lock order, can be taken next. + persistAndUnlock := func() error { + if !lockHeld || f.fs.lockMgr == nil { + return nil + } + if inoDirty { + if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { + log.Printf("[DEBUG] Write inode %d: persisting inode before unlocking failed: %v", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + return retryOnTreeConflict(err) + } + inoDirty = false + committedAllocs, committedCoW = len(allocatedExtents), len(cowDone) + } + releaseInodeLock() + return nil + } inodeWritten := false defer func() { if !inodeWritten { + // Discarding frees tree blocks, which needs the bitmap lock: drop the inode lock first. + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) } }() @@ -2201,46 +2293,39 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } - // Release inode lock before allocation to respect lock ordering: - // bitmap (6) must be acquired before inode (8) - // AllocateBlocks acquires bitmap lock internally - if lockHeld && f.fs.lockMgr != nil { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - lockHeld = false - } - - // Allocate block (acquires bitmap lock internally, then releases it) - ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) - if err != nil { - log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) - switch { - case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): - // Concurrent allocation by another node - return 0, errRetryWrite - case errors.Is(err, extent.ErrExtentTreeFull): - return 0, syscall.EFBIG - case errors.Is(err, extent.ErrCorruptTree): - return 0, syscall.EIO - default: - return 0, syscall.ENOSPC - } + // The inode is re-read after allocation, so persist changes made under the lock first. + if err := persistAndUnlock(); err != nil { + return 0, err } - // Re-acquire inode lock after allocation if f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to re-acquire lock after allocation: %v", f.inodeNum, err) - return 0, syscall.EIO + freshIno, newExt, releaseTree, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + if allocErr != nil { + log.Printf("[DEBUG] Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) + return 0, allocErr } lockHeld = true - - // Re-read inode after re-acquiring lock to verify state - freshIno, readErr := f.fs.inodeTbl.Read(f.inodeNum) - if readErr != nil { - log.Printf("[DEBUG] Write inode %d: re-read after allocation failed: %v", f.inodeNum, readErr) - return 0, syscall.EIO - } + releaseReservations = append(releaseReservations, releaseTree) ino = freshIno + inoDirty = true + ext = newExt + } else { + // Allocate block (acquires bitmap lock internally, then releases it) + ext, err = f.fs.extentMgr.AllocateBlocks(ctx, ino, blockNum, 1) + if err != nil { + log.Printf("[DEBUG] Write inode %d: AllocateBlocks at block %d failed: %v", f.inodeNum, blockNum, err) + switch { + case errors.Is(err, extent.ErrExtentOverlap), errors.Is(err, extent.ErrStaleTree): + // Concurrent allocation by another node + return 0, errRetryWrite + case errors.Is(err, extent.ErrExtentTreeFull): + return 0, syscall.EFBIG + case errors.Is(err, extent.ErrCorruptTree): + return 0, syscall.EIO + default: + return 0, syscall.ENOSPC + } + } } extOffset = 0 @@ -2315,6 +2400,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } if cowResult.WasShared { + inoDirty = true log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) @@ -2333,49 +2419,49 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Check if this is an unwritten extent (from fallocate). // If so, we need to clear the Unwritten flag on first write. - if ext.Flags&extent.FlagUnwritten != 0 { - // First write to unwritten block - need to clear flag - if !lockHeld && f.fs.lockMgr != nil { - // Acquire lock before modifying extent - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to acquire lock for unwritten: %v", f.inodeNum, err) - return 0, syscall.EIO - } - lockHeld = true - - // Re-read inode after acquiring lock - freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: re-read for unwritten failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - if freshIno.MetaGeneration != ino.MetaGeneration { - log.Printf("[DEBUG] Write inode %d: MetaGeneration changed during unwritten clear", - f.inodeNum) - return 0, errRetryWrite - } - ino = freshIno - - // Re-lookup extent - another node may have already cleared it - ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: extent disappeared during unwritten clear", - f.inodeNum) + if ext.Flags&extent.FlagUnwritten != 0 && f.fs.lockMgr != nil { + // Splitting the extent may allocate tree blocks, which needs the bitmap lock + // before the inode lock. Clear on an unlocked copy, then keep it only if no + // node wrote the inode before the lock was taken. + if err := persistAndUnlock(); err != nil { + return 0, err + } + cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ctx, ino, blockNum) + if clearErr != nil { + log.Printf("[DEBUG] Write inode %d: clearing Unwritten flag at block %d failed: %v", + f.inodeNum, blockNum, clearErr) + f.fs.extentMgr.DiscardPending(ctx, ino) + if errors.Is(clearErr, extent.ErrStaleTree) { return 0, errRetryWrite } + return 0, syscall.EIO } - - // Clear the Unwritten flag (if still set) + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + log.Printf("[DEBUG] Write inode %d: failed to acquire lock for unwritten: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, syscall.EIO + } + lockHeld = true + freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || freshIno.MetaGeneration != ino.MetaGeneration { + log.Printf("[DEBUG] Write inode %d: inode changed during unwritten clear (err=%v)", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, errRetryWrite + } + inoDirty = inoDirty || cleared + zeroFirst = cleared + if ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum); err != nil { + log.Printf("[DEBUG] Write inode %d: lookup after unwritten clear failed: %v", f.inodeNum, err) + return 0, syscall.EIO + } + } else if ext.Flags&extent.FlagUnwritten != 0 { cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ctx, ino, blockNum) if clearErr != nil { log.Printf("[DEBUG] Write inode %d: clearing Unwritten flag at block %d failed: %v", f.inodeNum, blockNum, clearErr) return 0, syscall.EIO } - if cleared { - log.Printf("[DEBUG] Write inode %d: cleared Unwritten flag for block %d", - f.inodeNum, blockNum) - } zeroFirst = cleared } @@ -2482,11 +2568,12 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Write inode %d: re-read inode failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, syscall.EIO } + // ino was read before the lock, so it can miss another node's extents. + if acquiredLockForSize { + ino = freshIno + } // Use the freshest size from disk, then calculate new size as max(fresh_size, our_write_end) newSize := freshIno.Size if offset > newSize { @@ -2540,9 +2627,12 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i ino.Size = newSize ino.Mtime = newMtime - // On a tree conflict the blocks allocated here are unreferenced. Close the - // intent first so journal replay cannot map them again, then free them. + // On a tree conflict the blocks allocated since the last mid-write persist are + // unreferenced. Close the intent first so journal replay cannot map them again, + // then free them. Blocks the persist committed stay mapped on disk. writeFailed := func(err error) error { + // The bitmap lock comes before the inode lock, so free only after dropping it. + releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) if !errors.Is(err, extent.ErrTreeConflict) { return syscall.EIO @@ -2554,12 +2644,12 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } var unused []uint64 - for _, alloc := range allocatedExtents { + for _, alloc := range allocatedExtents[committedAllocs:] { for i := uint64(0); i < uint64(alloc.count); i++ { unused = append(unused, alloc.physicalBlock+i) } } - for _, cow := range cowDone { + for _, cow := range cowDone[committedCoW:] { unused = append(unused, cow.NewPhysBlock) } for _, b := range unused { @@ -2567,16 +2657,26 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: freeing block %d after tree conflict failed: %v", f.inodeNum, b, err) } } + // The retry finds these blocks already copied, so release the old ones now. + for _, cow := range cowDone[:committedCoW] { + if _, err := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock); err != nil { + log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", + f.inodeNum, cow.OldPhysBlock, err) + } + } return errRetryWrite } + if testHookBeforeInodeCommit != nil { + if err := testHookBeforeInodeCommit(); err != nil { + return 0, writeFailed(err) + } + } + // Write inode - use WriteWithLockHeld if we hold the lock if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { log.Printf("[DEBUG] Write inode %d: WriteWithLockHeld failed: %v", f.inodeNum, err) - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } return 0, writeFailed(err) } } else { @@ -2588,10 +2688,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i inodeWritten = true - // Release the lock we acquired for size update - if acquiredLockForSize { - f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - } + releaseInodeLock() for _, cow := range cowDone { newCount, decErr := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock) @@ -3078,12 +3175,15 @@ func (f *File) allocate(ctx context.Context, off uint64, size uint64, mode uint3 } defer f.fs.lockMgr.Release(lock.LockTypeInode, uint64(f.inodeNum)) - // Re-read inode after lock to get latest state - ino, err = f.fs.inodeTbl.Read(f.inodeNum) + // ino holds the new extents, so keep it only if no node wrote the inode since it was read. + fresh, err := f.fs.inodeTbl.Read(f.inodeNum) if err != nil { log.Printf("[DEBUG] Allocate inode %d: re-read failed: %v", f.inodeNum, err) return syscall.EIO, nil } + if fresh.MetaGeneration != ino.MetaGeneration { + return 0, extent.ErrStaleTree + } } // Update file size if not KEEP_SIZE and new size is larger diff --git a/internal/goclusterfs/lock_commit_test.go b/internal/goclusterfs/lock_commit_test.go new file mode 100644 index 00000000..bbd41b6d --- /dev/null +++ b/internal/goclusterfs/lock_commit_test.go @@ -0,0 +1,99 @@ +package goclusterfs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/supporttools/GoClusterFS/internal/bitmap" + "github.com/supporttools/GoClusterFS/internal/dir" + "github.com/supporttools/GoClusterFS/internal/extent" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +// openLockedTreeFS is openTreeFS with a lock manager, as on a real mount. +func openLockedTreeFS(t *testing.T, path string) *FS { + t.Helper() + dev, err := storage.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { dev.Close() }) + lockPath := filepath.Join(t.TempDir(), "locks.img") + if err := os.WriteFile(lockPath, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(lockPath, 1<<20); err != nil { + t.Fatal(err) + } + lockDev, err := storage.Open(lockPath) + if err != nil { + t.Fatalf("open %s: %v", lockPath, err) + } + t.Cleanup(func() { lockDev.Close() }) + + var nodeUUID [16]byte + copy(nodeUUID[:], "testnode12345678") + lm := lock.NewManager(lockDev, 0, 1<<20, nodeUUID, 1) + bm := bitmap.NewRegion(dev, treeFSBitmapOffset, treeFSDataBlocks, lm, nodeUUID) + inodeTbl := inode.NewTable(dev, treeFSInodeOffset, 4096*inode.InodeSize, lm, nodeUUID) + extentMgr := extent.NewManager(dev, bm, lm, treeFSDataOffset) + inodeTbl.SetTreeFlusher(extentMgr) + return &FS{ + dev: dev, + lockMgr: lm, + bitmapMgr: bm, + inodeTbl: inodeTbl, + extentMgr: extentMgr, + dirMgr: dir.NewManager(dev, inodeTbl, extentMgr, lm, treeFSDataOffset), + nodeUUID: nodeUUID, + dataOffset: treeFSDataOffset, + } +} + +// A write over two new blocks commits the first one to the on-disk inode before +// it allocates the second. If the final inode write then fails, the retry must +// find the first block still allocated, not handed out again as the second. +func TestFailedWriteKeepsBlocksCommittedMidWrite(t *testing.T) { + fsys := openLockedTreeFS(t, newTreeFSImage(t)) + f := newRegularFile(t, fsys) + + conflicts := 1 + testHookBeforeInodeCommit = func() error { + if conflicts > 0 { + conflicts-- + return extent.ErrTreeConflict + } + return nil + } + defer func() { testHookBeforeInodeCommit = nil }() + + data := pattern(3, BlockSize+4096) + writeAt(t, f, data, 0) + if conflicts != 0 { + t.Fatal("the final inode write never ran") + } + + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatalf("read inode: %v", err) + } + seen := map[uint64]bool{} + for blk := uint64(0); blk < 2; blk++ { + ext, off, err := fsys.extentMgr.Lookup(ino, blk) + if err != nil { + t.Fatalf("lookup block %d: %v", blk, err) + } + phys := ext.PhysicalBlock + off + if seen[phys] { + t.Fatalf("logical blocks 0 and 1 both map to physical block %d", phys) + } + seen[phys] = true + if free, err := fsys.bitmapMgr.IsFree(phys); err != nil || free { + t.Fatalf("physical block %d is mapped but free in the bitmap (err=%v)", phys, err) + } + } + checkAt(t, f, data, 0) +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index edc3f4f4..8daf0578 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -202,9 +202,6 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { return nil, ErrInodeOutOfRange } - t.mu.Lock() - defer t.mu.Unlock() - // Acquire distributed lock if available if t.lockMgr != nil { if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inode.InodeNum), lock.LockModeExclusive); err != nil { @@ -213,6 +210,9 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { defer t.lockMgr.Release(lock.LockTypeInode, uint64(inode.InodeNum)) } + t.mu.Lock() + defer t.mu.Unlock() + return t.writeWithTree(inode) } @@ -311,11 +311,10 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { return nil, ErrInodeOutOfRange } - t.mu.Lock() - defer t.mu.Unlock() - // Re-read to verify slot is still free (another node may have claimed it) + t.mu.RLock() existing, err := t.readInodeUnlocked(num) + t.mu.RUnlock() if err != nil { return nil, fmt.Errorf("failed to read inode slot: %w", err) } @@ -331,6 +330,9 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { defer t.lockMgr.Release(lock.LockTypeInode, uint64(num)) } + t.mu.Lock() + defer t.mu.Unlock() + // Double-check after acquiring lock (another node could have won the race) existing, err = t.readInodeUnlocked(num) if err != nil { @@ -367,37 +369,16 @@ func (t *Table) newInodeInSlot(num uint32, free *Inode) *Inode { // Deprecated: Use FindFreeSlot() + ClaimSlot() for operations that need to allocate // blocks before claiming the inode (to respect lock ordering: BlockBitmap before Inode). func (t *Table) Alloc(ctx context.Context) (*Inode, error) { - t.mu.Lock() - defer t.mu.Unlock() - - // Scan for a free inode starting from InodeFirstUsable - for num := InodeFirstUsable; uint64(num) < t.totalInodes; num++ { - inode, err := t.readInodeUnlocked(num) + for { + num, err := t.FindFreeSlot() if err != nil { - continue // Skip unreadable inodes + return nil, err } - - // Check if inode is free (Magic == 0) - if inode.Magic == 0 { - newInode := t.newInodeInSlot(num, inode) - - // Acquire distributed lock if available - if t.lockMgr != nil { - if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(num), lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire inode lock: %w", err) - } - defer t.lockMgr.Release(lock.LockTypeInode, uint64(num)) - } - - if err := t.writeInodeUnlocked(newInode); err != nil { - return nil, fmt.Errorf("failed to write new inode: %w", err) - } - - return newInode, nil + ino, err := t.ClaimSlot(ctx, num) + if !errors.Is(err, ErrInodeInUse) { + return ino, err } } - - return nil, ErrInodeFull } // Free marks an inode as free. @@ -410,9 +391,6 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error { return fmt.Errorf("cannot free reserved inode %d", inodeNum) } - t.mu.Lock() - defer t.mu.Unlock() - // Acquire distributed lock if available if t.lockMgr != nil { if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inodeNum), lock.LockModeExclusive); err != nil { @@ -421,6 +399,9 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error { defer t.lockMgr.Release(lock.LockTypeInode, uint64(inodeNum)) } + t.mu.Lock() + defer t.mu.Unlock() + // Read current inode inode, err := t.readInodeUnlocked(inodeNum) if err != nil { diff --git a/internal/inode/lock_test.go b/internal/inode/lock_test.go new file mode 100644 index 00000000..137a51b1 --- /dev/null +++ b/internal/inode/lock_test.go @@ -0,0 +1,60 @@ +package inode + +import ( + "context" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/lock" +) + +// A directory listing holds the directory's shared lock and then reads the inode +// table, while Write holds the table mutex and asks for the exclusive lock. +func TestWriteDoesNotHangBehindSameNodeReader(t *testing.T) { + dev, cleanup := createTestDevice(t, 16<<20) + defer cleanup() + var uuid [16]byte + copy(uuid[:], "node-a") + lm := lock.NewManager(dev, 8<<20, 1<<20, uuid, 1) + table := NewTable(dev, 0, 4<<20, lm, uuid) + ctx := context.Background() + dir, err := table.Alloc(ctx) + if err != nil { + t.Fatalf("Alloc: %v", err) + } + key := uint64(dir.InodeNum) + + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + t.Fatalf("Acquire shared: %v", err) + } + writeDone := make(chan error, 1) + go func() { + wctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + writeDone <- table.Write(wctx, dir) + }() + time.Sleep(50 * time.Millisecond) + + readDone := make(chan error, 1) + go func() { + if err := lm.Acquire(ctx, lock.LockTypeInode, key, lock.LockModeShared); err != nil { + readDone <- err + return + } + _, err := table.Read(dir.InodeNum) + lm.Release(lock.LockTypeInode, key) + readDone <- err + }() + lm.Release(lock.LockTypeInode, key) + + for name, done := range map[string]chan error{"Write": writeDone, "listing": readDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("%s: %v", name, err) + } + case <-time.After(2 * time.Second): + t.Fatalf("%s did not finish: same-node reader and writer deadlocked", name) + } + } +} diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 042fed5f..6452f652 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -206,6 +206,11 @@ type Manager struct { nodeSlot int // This node's slot heldLocks map[lockKey]*heldLock + // Several keys can hash to one record; the record stays held until the last key is released. + records map[uint64]*heldRecord + // gates serialize this node's on-disk operations per record. + gates map[uint64]*sync.Mutex + // Callback for cache invalidation onAcquire OnAcquireFunc @@ -233,6 +238,21 @@ type heldLock struct { goroutines map[uint64]int // Goroutine ID -> reference count for that goroutine } +type heldRecord struct { + mode LockMode + keys int + renewed time.Time + lost bool +} + +func covers(held, want LockMode) bool { + return held == LockModeExclusive || held == want +} + +// renewAfter is the age of an exclusive hold after which a key joining the record +// rewrites AcquireTime, so a record kept busy by overlapping keys never looks stale. +var renewAfter = StaleThreshold / 3 + // Errors var ( ErrLockTimeout = errors.New("lock acquisition timed out") @@ -240,6 +260,7 @@ var ( ErrNotLockOwner = errors.New("not the lock owner") ErrDeadlockRisk = errors.New("lock order violation would cause deadlock") ErrInvalidRecord = errors.New("invalid lock record") + ErrLockLost = errors.New("lock record was taken over by another node") ) // NewManager creates a new lock manager. @@ -251,6 +272,8 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(HintCacheSize), } } @@ -265,6 +288,8 @@ func NewManagerWithHintCacheSize(dev *storage.Device, offset, size uint64, nodeU nodeUUID: nodeUUID, nodeSlot: nodeSlot, heldLocks: make(map[lockKey]*heldLock), + records: make(map[uint64]*heldRecord), + gates: make(map[uint64]*sync.Mutex), hintCache: newHintLRU(hintCacheSize), } } @@ -303,33 +328,25 @@ func (m *Manager) HintStats() (hits, misses uint64, hitRate float64) { // subsequent goroutines increment a local reference count. This allows // concurrent FUSE operations from the same node to share locks efficiently // while still providing cross-node serialization. +// +// Resources that hash to the same record share it: a node holding the record +// for one resource excludes other nodes from every resource in that record. func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uint64, mode LockMode) error { key := lockKey{Type: lockType, ResourceID: resourceID} - + recordID := m.hashToRecord(lockType, resourceID) gid := getGoroutineID() m.mu.Lock() - // Check if we already hold this lock (same node) - just increment refCount - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - reusing existing lock (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) + if m.reuseHeldKeyLocked(key, mode, gid) { m.mu.Unlock() return nil } - - // Check lock ordering to prevent deadlocks if err := m.checkLockOrder(lockType); err != nil { m.mu.Unlock() return err } m.mu.Unlock() - // Calculate record position (hash-based) - recordID := m.hashToRecord(lockType, resourceID) retryCount := 0 startTime := time.Now() @@ -338,63 +355,30 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin for { // Always attempt acquire first, then check context after failure // This ensures we try at least once even if context is nearly expired - acquired, err := m.tryAcquire(recordID, lockType, resourceID, mode) + held, added, err := m.attemptAcquire(key, recordID, mode, gid) if err != nil { - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - tryAcquire error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - attempt error: %v", lockType, resourceID, err) return err } - if acquired { - m.mu.Lock() - // Double-check in case another goroutine acquired while we were trying - if held, ok := m.heldLocks[key]; ok { - // Another goroutine beat us - just increment refCount - held.refCount++ - if held.goroutines == nil { - held.goroutines = make(map[uint64]int) - } - held.goroutines[gid]++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - late arrival, reusing (refCount=%d, gid=%d)", lockType, resourceID, held.refCount, gid) - m.mu.Unlock() - // Release the on-disk lock we just acquired since we'll share the existing one - m.releaseRecord(recordID) - return nil - } - m.heldLocks[key] = &heldLock{ - Mode: mode, - RecordID: recordID, - refCount: 1, - goroutines: map[uint64]int{gid: 1}, - } - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired distributed lock (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) + if held { + log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - acquired (retries=%d, dur=%v, gid=%d)", lockType, resourceID, retryCount, time.Since(startTime), gid) // Trigger cache invalidation callback for exclusive locks. // We do this before returning so the caller reads fresh data. // Only exclusive locks trigger invalidation since shared locks // are for reading and can safely use cached values. + m.mu.Lock() onAcquire := m.onAcquire m.mu.Unlock() - - if onAcquire != nil && mode == LockModeExclusive { + if added && onAcquire != nil && mode == LockModeExclusive { onAcquire(lockType, resourceID, mode) } return nil } - // Check again if another goroutine acquired while we were waiting - m.mu.Lock() - if held, ok := m.heldLocks[key]; ok { - held.refCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - piggyback on existing (refCount=%d)", lockType, resourceID, held.refCount) - m.mu.Unlock() - return nil - } - m.mu.Unlock() - retryCount++ - log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - loop iteration, retryCount now %d", lockType, resourceID, retryCount) - // Check context cancellation, but allow at least 3 retries for same-node - // CAS races where the other goroutine is still adding to heldLocks + // Check context cancellation, but allow at least 3 retries if retryCount >= 3 { select { case <-ctx.Done(): @@ -416,6 +400,107 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin } } +// attemptAcquire makes one acquisition attempt. added reports whether this +// call registered the key rather than joining another goroutine's hold. +// +// A key joins this node's hold on its record when the hold's mode covers the +// request. An exclusive request waits for this node's own readers, so callers +// must not hold a local mutex across Acquire that those readers may need. +func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64) (held, added bool, err error) { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + if rec, ok := m.records[recordID]; ok { + switch { + case rec.lost: + m.mu.Unlock() + return false, false, ErrLockLost + case !covers(rec.mode, mode): + selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) + m.mu.Unlock() + if selfDeadlock { + return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) + } + return false, false, nil + } + renew := rec.mode == LockModeExclusive && time.Since(rec.renewed) > renewAfter + m.mu.Unlock() + if renew { + if err := m.renewRecord(recordID, rec); err != nil { + return false, false, err + } + } + m.mu.Lock() + if held, ok := m.heldLocks[key]; ok { + held.refCount++ + held.goroutines[gid]++ + m.mu.Unlock() + return true, false, nil + } + rec.keys++ + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil + } + m.mu.Unlock() + + acquired, err := m.tryAcquire(recordID, key.Type, key.ResourceID, mode) + if err != nil || !acquired { + return false, false, err + } + + m.mu.Lock() + m.records[recordID] = &heldRecord{mode: mode, keys: 1, renewed: time.Now()} + m.addHeldKeyLocked(key, recordID, mode, gid) + m.mu.Unlock() + return true, true, nil +} + +func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { + held, ok := m.heldLocks[key] + if !ok { + return false + } + rec := m.records[held.RecordID] + if rec.lost || !covers(rec.mode, mode) || (rec.mode == LockModeExclusive && time.Since(rec.renewed) > renewAfter) { + return false + } + held.refCount++ + held.goroutines[gid]++ + return true +} + +func (m *Manager) addHeldKeyLocked(key lockKey, recordID uint64, mode LockMode, gid uint64) { + m.heldLocks[key] = &heldLock{ + Mode: mode, + RecordID: recordID, + refCount: 1, + goroutines: map[uint64]int{gid: 1}, + } +} + +func (m *Manager) goroutineHoldsRecordLocked(recordID uint64, gid uint64) bool { + for _, held := range m.heldLocks { + if held.RecordID == recordID && held.goroutines[gid] > 0 { + return true + } + } + return false +} + +func (m *Manager) recordGate(recordID uint64) *sync.Mutex { + m.mu.Lock() + defer m.mu.Unlock() + gate, ok := m.gates[recordID] + if !ok { + gate = &sync.Mutex{} + m.gates[recordID] = gate + } + return gate +} + // Release releases a held lock. // With reference counting, this decrements the local count and only releases // the distributed lock when the last goroutine releases. @@ -431,36 +516,51 @@ func (m *Manager) Release(lockType LockType, resourceID uint64) error { return ErrNotLockOwner } - // Decrement goroutine-specific count - if held.goroutines != nil { - held.goroutines[gid]-- - if held.goroutines[gid] <= 0 { - delete(held.goroutines, gid) - } + held.goroutines[gid]-- + if held.goroutines[gid] <= 0 { + delete(held.goroutines, gid) } // Decrement overall reference count held.refCount-- if held.refCount > 0 { - // Other goroutines still using the lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - decremented refCount to %d", lockType, resourceID, gid, held.refCount) m.mu.Unlock() return nil } - // Last reference - release the distributed lock - log.Printf("[DEBUG] lock.Release: type=%d res=%d gid=%d - last ref, releasing distributed lock", lockType, resourceID, gid) delete(m.heldLocks, key) recordID := held.RecordID m.mu.Unlock() - err := m.releaseRecord(recordID) + err := m.dropRecordKey(recordID) if err != nil { - log.Printf("[DEBUG] lock.Release: type=%d res=%d - releaseRecord error: %v", lockType, resourceID, err) + log.Printf("[DEBUG] lock.Release: type=%d res=%d - release error: %v", lockType, resourceID, err) } return err } +func (m *Manager) dropRecordKey(recordID uint64) error { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + rec, ok := m.records[recordID] + if !ok { + m.mu.Unlock() + return ErrNotLockOwner + } + rec.keys-- + if rec.keys > 0 { + m.mu.Unlock() + return nil + } + delete(m.records, recordID) + m.mu.Unlock() + + return m.releaseRecord(recordID, rec.mode) +} + // tryAcquire attempts a single lock acquisition. // First tries an optimistic path using cached sequence hints, then falls back // to reading the current record if the hint is stale. @@ -499,40 +599,29 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint now := time.Now() - // Check if lock is free or can be acquired - canAcquire := false - reason := "unknown" + // A record held for a colliding resource is still held; taking it over lets two nodes hold it. + var sharedCount uint32 + acquireTime := now.Unix() + reason := "" switch { case record.Mode == LockModeNone: - // Lock is free - canAcquire = true reason = "free" - case record.Type != lockType || record.ResourceID != resourceID: - // Hash collision - different resource, treat as free - // In production, we'd use a more sophisticated collision resolution - canAcquire = true - reason = "hash_collision" - case m.isLockStale(record): - // Lock is stale (owner died) - canAcquire = true + // Owner presumed dead; its shared holders are gone too. reason = "stale" case mode == LockModeShared && record.Mode == LockModeShared: - // Adding another shared holder - canAcquire = true reason = "shared_add" + sharedCount = record.SharedCount + // Keep the first reader's time: a count leaked by a lost CAS update must still go stale. + acquireTime = record.AcquireTime - case record.OwnerUUID == m.nodeUUID: - // Same node owns the lock - this can happen due to a race between - // Release clearing heldLocks and releaseRecord clearing on-disk. - // Safe to re-acquire since the previous holder from this node is gone. - canAcquire = true - reason = "same_node_reacquire" - } + case record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID: + // Leftover from a failed release or an earlier mount; this node holds no key on it. + reason = "leftover_from_this_node" - if !canAcquire { + default: log.Printf("[DEBUG] lock.tryAcquire: type=%d res=%d - cannot acquire, held by node=%x mode=%d", lockType, resourceID, record.OwnerUUID[:8], record.Mode) return false, nil } @@ -547,11 +636,11 @@ func (m *Manager) tryAcquire(recordID uint64, lockType LockType, resourceID uint ResourceID: resourceID, OwnerUUID: m.nodeUUID, OwnerSlot: uint32(m.nodeSlot), - AcquireTime: now.Unix(), + AcquireTime: acquireTime, Sequence: record.Sequence + 1, } if mode == LockModeShared { - newRecord.SharedCount = record.SharedCount + 1 + newRecord.SharedCount = sharedCount + 1 } // CAS the record @@ -603,58 +692,77 @@ func (m *Manager) tryAcquireOptimistic(recordID uint64, lockType LockType, resou return acquired, newSeq, nil } -// releaseRecord releases a lock record. -// After successful release, caches the new sequence number to enable -// optimistic acquisition on the next lock attempt. -func (m *Manager) releaseRecord(recordID uint64) error { - record, err := m.readRecord(recordID) - if err != nil { - return err - } - - // Verify we own it - if record.OwnerUUID != m.nodeUUID { - // For shared locks, just decrement count - if record.Mode == LockModeShared && record.SharedCount > 1 { - newRecord := *record - newRecord.SharedCount-- - newRecord.Sequence++ - success, err := m.casRecord(recordID, record, &newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeShared, // Still shared (decremented) - UpdateAt: time.Now(), - }) - } +// renewRecord rewrites AcquireTime on a record this node holds exclusively. It +// runs under the record gate. If another node took the record over as stale, +// the hold is marked lost and no new key may join it. +func (m *Manager) renewRecord(recordID uint64, rec *heldRecord) error { + for { + record, err := m.readRecord(recordID) + if err != nil { + return err + } + if record.Mode != LockModeExclusive || record.OwnerUUID != m.nodeUUID { + log.Printf("[WARN] lock: record %d was taken over while this node held it", recordID) + m.mu.Lock() + rec.lost = true + m.mu.Unlock() + return ErrLockLost + } + renewed := *record + renewed.AcquireTime = time.Now().Unix() + renewed.Sequence++ + ok, err := m.casRecord(recordID, record, &renewed) + if err != nil { + return err + } + if ok { + m.mu.Lock() + rec.renewed = time.Now() + m.mu.Unlock() return nil } - return ErrNotLockOwner } +} - // Clear the record - newRecord := &Record{ - Sequence: record.Sequence + 1, - } +// releaseRecord releases this node's hold on a lock record. +// After successful release, caches the new sequence number to enable +// optimistic acquisition on the next lock attempt. +func (m *Manager) releaseRecord(recordID uint64, mode LockMode) error { + for { + record, err := m.readRecord(recordID) + if err != nil { + return err + } - success, err := m.casRecord(recordID, record, newRecord) - if err != nil { - return err - } - if success { - // Cache the new sequence for optimistic re-acquisition - // Mode is None since lock is now free - m.hintCache.Put(recordID, &lockHint{ - Sequence: newRecord.Sequence, - Mode: LockModeNone, - UpdateAt: time.Now(), - }) + var newRecord *Record + switch { + case mode == LockModeExclusive && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID, + mode == LockModeShared && record.Mode == LockModeShared && record.SharedCount <= 1: + newRecord = &Record{Sequence: record.Sequence + 1} + case mode == LockModeShared && record.Mode == LockModeShared: + // OwnerUUID names only the latest reader, so readers release by count. + next := *record + next.SharedCount-- + next.Sequence++ + newRecord = &next + default: + // Another node reclaimed the record after it went stale. + return ErrNotLockOwner + } + + success, err := m.casRecord(recordID, record, newRecord) + if err != nil { + return err + } + if success { + m.hintCache.Put(recordID, &lockHint{ + Sequence: newRecord.Sequence, + Mode: newRecord.Mode, + UpdateAt: time.Now(), + }) + return nil + } } - return nil } // isLockStale checks if a lock's owner is dead. diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go new file mode 100644 index 00000000..71063153 --- /dev/null +++ b/internal/lock/safety_test.go @@ -0,0 +1,395 @@ +package lock + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/storage" +) + +func newTestNode(dev *storage.Device, offset, size uint64, n int) *Manager { + var uuid [16]byte + copy(uuid[:], fmt.Sprintf("test-node-%06d", n)) + return NewManager(dev, offset, size, uuid, n) +} + +func acquireWithin(m *Manager, d time.Duration, lt LockType, id uint64, mode LockMode) error { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + return m.Acquire(ctx, lt, id, mode) +} + +func requireBlocked(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + err := acquireWithin(m, 50*time.Millisecond, lt, id, mode) + if err == nil { + m.Release(lt, id) + t.Fatalf("node %d acquired type=%d id=%d mode=%d while another holder should exclude it", m.nodeSlot, lt, id, mode) + } + if !errors.Is(err, ErrLockTimeout) { + t.Fatalf("Acquire: got %v, want ErrLockTimeout", err) + } +} + +func requireAcquired(t *testing.T, m *Manager, lt LockType, id uint64, mode LockMode) { + t.Helper() + if err := acquireWithin(m, time.Second, lt, id, mode); err != nil { + t.Fatalf("node %d Acquire type=%d id=%d mode=%d: %v", m.nodeSlot, lt, id, mode, err) + } +} + +// ageRecord rewrites the lock's record as if it was taken age ago and never renewed. +func ageRecord(t *testing.T, m *Manager, lt LockType, id uint64, age time.Duration) { + t.Helper() + recordID := m.hashToRecord(lt, id) + rec, err := m.readRecord(recordID) + if err != nil { + t.Fatalf("readRecord: %v", err) + } + rec.AcquireTime = time.Now().Add(-age).Unix() + buf := storage.GetBuffer(RecordSize) + defer storage.PutBuffer(buf) + copy(buf.Bytes(), serializeRecord(rec)) + if _, err := m.dev.WriteAt(buf.Bytes(), int64(m.offset)+int64(recordID)*RecordSize); err != nil { + t.Fatalf("write record: %v", err) + } +} + +const pastStale = StaleThreshold + 2*time.Second + +func TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + requireBlocked(t, b, LockTypeInode, 2, LockModeExclusive) +} + +func TestNodeHoldsCollidingResourcesUntilLastRelease(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + requireAcquired(t, a, LockTypeInode, 7, LockModeExclusive) + if err := a.Release(LockTypeInode, 7); err != nil { + t.Fatalf("Release inode: %v", err) + } + requireBlocked(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + + if err := a.Release(LockTypeBlockBitmap, 0); err != nil { + t.Fatalf("Release bitmap: %v", err) + } + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) +} + +func TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + const workers = 4 + for iter := 0; iter < 30; iter++ { + start := make(chan struct{}) + release := make(chan struct{}) + var acquired, done sync.WaitGroup + errs := make(chan error, workers) + acquired.Add(workers) + done.Add(workers) + for w := 0; w < workers; w++ { + go func() { + defer done.Done() + <-start + err := acquireWithin(a, time.Second, LockTypeInode, 42, LockModeExclusive) + acquired.Done() + if err != nil { + errs <- err + return + } + <-release + if err := a.Release(LockTypeInode, 42); err != nil { + errs <- err + } + }() + } + close(start) + acquired.Wait() + requireBlocked(t, b, LockTypeInode, 42, LockModeExclusive) + close(release) + done.Wait() + close(errs) + for err := range errs { + t.Fatalf("iteration %d: %v", iter, err) + } + } +} + +func TestSharedLockHeldAfterAnotherReaderReleases(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 5, LockModeShared) + requireAcquired(t, b, LockTypeInode, 5, LockModeShared) + if err := b.Release(LockTypeInode, 5); err != nil { + t.Fatalf("b Release: %v", err) + } + requireBlocked(t, c, LockTypeInode, 5, LockModeExclusive) + + if err := a.Release(LockTypeInode, 5); err != nil { + t.Fatalf("a Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 5, LockModeExclusive) +} + +func TestExclusiveWaitsForSameNodeReaders(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + + writer := func() error { + errc := make(chan error, 1) + go func() { + err := acquireWithin(a, 50*time.Millisecond, LockTypeInode, 9, LockModeExclusive) + if err == nil { + err = a.Release(LockTypeInode, 9) + } + errc <- err + }() + return <-errc + } + + if err := writer(); !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while a reader on the same node holds the lock: got %v, want ErrLockTimeout", err) + } + if err := a.Release(LockTypeInode, 9); err != nil { + t.Fatalf("Release shared: %v", err) + } + if err := writer(); err != nil { + t.Fatalf("exclusive Acquire after readers left: %v", err) + } +} + +func TestSameGoroutineCannotUpgradeSharedToExclusive(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + + requireAcquired(t, a, LockTypeInode, 9, LockModeShared) + err := acquireWithin(a, time.Second, LockTypeInode, 9, LockModeExclusive) + if !errors.Is(err, ErrDeadlockRisk) { + t.Fatalf("upgrade in the same goroutine: got %v, want ErrDeadlockRisk", err) + } +} + +func TestReadersOnTwoNodesCannotBothWrite(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + requireAcquired(t, a, LockTypeInode, 77, LockModeShared) + requireAcquired(t, b, LockTypeInode, 77, LockModeShared) + + errs := make(chan error, 2) + for _, m := range []*Manager{a, b} { + go func(m *Manager) { + err := acquireWithin(m, 50*time.Millisecond, LockTypeInode, 77, LockModeExclusive) + if err == nil { + m.Release(LockTypeInode, 77) + } + errs <- err + }(m) + } + for i := 0; i < 2; i++ { + if err := <-errs; !errors.Is(err, ErrLockTimeout) { + t.Fatalf("exclusive Acquire while both nodes read: got %v, want ErrLockTimeout", err) + } + } +} + +func TestJoiningKeyRenewsBusyRecord(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + defer func(d time.Duration) { renewAfter = d }(renewAfter) + renewAfter = 0 + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 1, pastStale) + requireAcquired(t, a, LockTypeInode, 2, LockModeExclusive) + if err := a.Release(LockTypeInode, 1); err != nil { + t.Fatalf("Release: %v", err) + } + + requireBlocked(t, b, LockTypeInode, 3, LockModeExclusive) +} + +func TestStaleLockIsReclaimedFromPausedHolder(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + + requireAcquired(t, a, LockTypeBlockBitmap, 0, LockModeExclusive) + ageRecord(t, a, LockTypeBlockBitmap, 0, pastStale) + + requireAcquired(t, b, LockTypeBlockBitmap, 0, LockModeExclusive) + if held := a.HeldLocks(); len(held) != 1 { + t.Fatalf("paused holder's view: got %v, want it to still list the lock (no fencing)", held) + } +} + +func TestPausedHolderReleaseKeepsReclaimedLock(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + b := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, a, LockTypeInode, 3, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 3, pastStale) + requireAcquired(t, b, LockTypeInode, 3, LockModeExclusive) + + if err := a.Release(LockTypeInode, 3); !errors.Is(err, ErrNotLockOwner) { + t.Fatalf("paused holder Release: got %v, want ErrNotLockOwner", err) + } + requireBlocked(t, c, LockTypeInode, 3, LockModeExclusive) +} + +func TestStaleSharedLockReclaimResetsReaderCount(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, 1<<20, 1) + dead := newTestNode(dev, 0, 1<<20, 2) + c := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, dead, LockTypeInode, 4, LockModeShared) + ageRecord(t, dead, LockTypeInode, 4, pastStale) + requireAcquired(t, a, LockTypeInode, 4, LockModeShared) + if err := a.Release(LockTypeInode, 4); err != nil { + t.Fatalf("Release: %v", err) + } + requireAcquired(t, c, LockTypeInode, 4, LockModeExclusive) +} + +func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { + const ( + guard = 4 * RecordSize + regionSize = 16 * RecordSize + resources = 1000 + cycles = 1500 + ) + dev, cleanup := createTestDevice(t, guard+regionSize+guard) + defer cleanup() + + sentinel := storage.NewAlignedBuffer(guard) + for i := range sentinel.Bytes() { + sentinel.Bytes()[i] = 0xA5 + } + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.WriteAt(sentinel.Bytes(), off); err != nil { + t.Fatalf("write sentinel: %v", err) + } + } + + nodes := []*Manager{ + newTestNode(dev, guard, regionSize, 1), + newTestNode(dev, guard, regionSize, 2), + } + var wg sync.WaitGroup + errs := make(chan error, len(nodes)) + for n, m := range nodes { + wg.Add(1) + go func(n int, m *Manager) { + defer wg.Done() + for i := 0; i < cycles; i++ { + id := uint64((i*7 + n*13) % resources) + lt := LockTypeInode + if i%3 == 0 { + lt = LockTypeExtent + } + if err := acquireWithin(m, 5*time.Second, lt, id, LockModeExclusive); err != nil { + errs <- fmt.Errorf("node %d cycle %d: %w", n, i, err) + return + } + m.Release(lt, id) + } + }(n, m) + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + check := newTestNode(dev, guard, regionSize, 3) + for id := uint64(0); id < resources; id++ { + for _, lt := range []LockType{LockTypeInode, LockTypeExtent} { + requireAcquired(t, check, lt, id, LockModeExclusive) + if err := check.Release(lt, id); err != nil { + t.Fatalf("Release type=%d id=%d: %v", lt, id, err) + } + } + } + + got := storage.NewAlignedBuffer(guard) + for _, off := range []int64{0, guard + regionSize} { + if _, err := dev.ReadAt(got.Bytes(), off); err != nil { + t.Fatalf("read sentinel: %v", err) + } + if !bytes.Equal(got.Bytes(), sentinel.Bytes()) { + t.Fatalf("lock manager wrote outside its region near offset %d", off) + } + } +} + +func TestNewReadersDoNotKeepLeakedSharedLockFresh(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + leaked := newTestNode(dev, 0, 1<<20, 1) + reader := newTestNode(dev, 0, 1<<20, 2) + writer := newTestNode(dev, 0, 1<<20, 3) + + requireAcquired(t, leaked, LockTypeInode, 2, LockModeShared) + ageRecord(t, leaked, LockTypeInode, 2, StaleThreshold-2*time.Second) + + requireAcquired(t, reader, LockTypeInode, 2, LockModeShared) + if err := reader.Release(LockTypeInode, 2); err != nil { + t.Fatalf("reader Release: %v", err) + } + time.Sleep(3 * time.Second) + + requireAcquired(t, writer, LockTypeInode, 2, LockModeExclusive) +} + +func TestJoinFailsAfterRecordTakenOver(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + defer func(d time.Duration) { renewAfter = d }(renewAfter) + renewAfter = 0 + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + ageRecord(t, a, LockTypeInode, 1, pastStale) + requireAcquired(t, b, LockTypeInode, 2, LockModeExclusive) + + if err := acquireWithin(a, time.Second, LockTypeInode, 3, LockModeExclusive); !errors.Is(err, ErrLockLost) { + t.Fatalf("joining a record another node took over: got %v, want ErrLockLost", err) + } +} diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 7c221f91..6863179c 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -130,9 +130,6 @@ func (r *Region) Get(ctx context.Context, blockNum uint64) (Entry, error) { return Entry{}, ErrBlockOutOfRange } - r.mu.RLock() - defer r.mu.RUnlock() - // Acquire distributed lock if available (shared mode for read) if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeShared); err != nil { @@ -141,6 +138,9 @@ func (r *Region) Get(ctx context.Context, blockNum uint64) (Entry, error) { defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.RLock() + defer r.mu.RUnlock() + return r.readEntry(blockNum) } @@ -151,9 +151,6 @@ func (r *Region) Increment(ctx context.Context, blockNum uint64) (uint32, error) return 0, ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -162,6 +159,9 @@ func (r *Region) Increment(ctx context.Context, blockNum uint64) (uint32, error) defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return 0, err @@ -188,9 +188,6 @@ func (r *Region) Decrement(ctx context.Context, blockNum uint64) (uint32, error) return 0, ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -199,6 +196,9 @@ func (r *Region) Decrement(ctx context.Context, blockNum uint64) (uint32, error) defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return 0, err @@ -224,9 +224,6 @@ func (r *Region) Set(ctx context.Context, blockNum uint64, count uint32, flags E return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -235,6 +232,9 @@ func (r *Region) Set(ctx context.Context, blockNum uint64, count uint32, flags E defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry := Entry{ RefCount: count, Flags: flags, @@ -248,9 +248,6 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags return ErrBlockOutOfRange } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -259,6 +256,9 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + entry, err := r.readEntry(blockNum) if err != nil { return err @@ -282,9 +282,6 @@ func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint } } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -293,6 +290,9 @@ func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + results := make([]uint32, len(blockNums)) // Group blocks by sector for efficient I/O @@ -344,9 +344,6 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint } } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { @@ -355,6 +352,9 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } + r.mu.Lock() + defer r.mu.Unlock() + results := make([]uint32, len(blockNums)) // Group blocks by sector for efficient I/O diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 0de4079d..ddc0f9a8 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -261,9 +261,6 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, return nil, ErrNameTooLong } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { // Use lock type for snapshot registry (we'll use a constant ID since registry is single) @@ -273,6 +270,9 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + // Find a free slot slot, err := r.findFreeSlot() if err != nil { @@ -420,9 +420,6 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -431,6 +428,9 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { @@ -451,9 +451,6 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -462,6 +459,9 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + // Find and update the descriptor for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) @@ -486,9 +486,6 @@ func (r *Registry) SetRefCountJobID(ctx context.Context, snapshotID, jobID uint6 return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -497,6 +494,9 @@ func (r *Registry) SetRefCountJobID(ctx context.Context, snapshotID, jobID uint6 defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { @@ -524,9 +524,6 @@ func (r *Registry) Remove(ctx context.Context, snapshotID uint64) error { return ErrInvalidID } - r.mu.Lock() - defer r.mu.Unlock() - // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeSnapshot, 0, lock.LockModeExclusive); err != nil { @@ -535,6 +532,9 @@ func (r *Registry) Remove(ctx context.Context, snapshotID uint64) error { defer r.lockMgr.Release(lock.LockTypeSnapshot, 0) } + r.mu.Lock() + defer r.mu.Unlock() + for slot := uint64(0); slot < r.maxDescriptors; slot++ { desc, err := r.readDescriptor(slot) if err != nil { diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 41dd9129..adca04ae 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -1919,6 +1919,46 @@ test_concurrent_writes() { return 0 } +# Two nodes write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other node's extent. +test_concurrent_first_write_same_block() { + log "Testing concurrent first writes to the same block from two nodes..." + local writer0="${NODE_ARRAY[0]}" + local writer1="${NODE_ARRAY[1]}" + local reader="${NODE_ARRAY[2]}" + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + exec_on_node "$writer0" touch "/mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at "$writer0" "first_write_$i.bin" A "$size" "$offset0" & + write_pattern_at "$writer1" "first_write_$i.bin" B "$size" "$offset1" & + wait + done + + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + restart_node "$reader" + wait_node_healthy "$reader" 30 || return 1 + + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at "$reader" "first_write_$i.bin" A "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at "$reader" "first_write_$i.bin" B "$offset1" 4096 || lost=$((lost + 1)) + done + if [ "$lost" -gt 0 ]; then + log "ERROR: $lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log "All concurrent first writes survived" + return 0 +} + # Test 2: Container kill and recovery test_container_kill_recovery() { log "Testing container kill and recovery..." @@ -2784,6 +2824,7 @@ else run_test "Concurrent File Creation" test_concurrent_file_creation run_test "Concurrent Directory Ops" test_concurrent_directory_ops fi +run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Container Kill Recovery" test_container_kill_recovery run_test "Journal Recovery Multi-Node" test_journal_recovery_multinode run_test "Restart Keeps Peer Journal" test_restart_keeps_peer_journal diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index f421b0c4..c6b23f27 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -1531,3 +1531,77 @@ func TestStressRandomFailures(t *testing.T) { t.Logf("Stress test completed: %d files created", filesCreated) } + +// TestMultiNodeConcurrentFirstWriteToSameBlock has two nodes write different +// ranges of a new file's first block at the same time. Each node must take the +// inode lock around its read-modify-write of the extents, or the later inode +// write drops the other node's extent and its data. +func TestMultiNodeConcurrentFirstWriteToSameBlock(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 100) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + if err := cluster.WaitAllMounted(15 * time.Second); err != nil { + t.Fatalf("Nodes failed to mount: %v", err) + } + + const files = 20 + name := func(i int) string { return fmt.Sprintf("first_write_%02d.bin", i) } + for i := 0; i < files; i++ { + if err := cluster.Node(0).WriteFile(name(i), nil); err != nil { + t.Fatalf("Failed to create %s: %v", name(i), err) + } + } + // Outlast the kernel's 1s attribute cache so node 1 sees the files. + time.Sleep(1500 * time.Millisecond) + + // Offsets past 64KB keep the files off the SFB path. + const offset0, offset1 = 256 * 1024, 512 * 1024 + data0 := bytes.Repeat([]byte("N0"), 32*1024) + data1 := bytes.Repeat([]byte("N1"), 32*1024) + + for i := 0; i < files; i++ { + errs := cluster.RunConcurrent([]int{0, 1}, func(idx int, node *MountedFS) error { + if idx == 0 { + return node.WriteFileAt(name(i), data0, offset0) + } + return node.WriteFileAt(name(i), data1, offset1) + }) + for idx, err := range errs { + if err != nil { + t.Fatalf("%s: node %d write failed: %v", name(i), idx, err) + } + } + } + + // Read from a freshly mounted node so no cached extents hide the result. + cluster.StopAllNodes() + if err := cluster.StartNode(0, false); err != nil { + t.Fatalf("Failed to remount node 0: %v", err) + } + + lost := 0 + for i := 0; i < files; i++ { + got, err := cluster.Node(0).ReadFile(name(i)) + if err != nil { + t.Fatalf("Failed to read %s: %v", name(i), err) + } + if len(got) < offset1+len(data1) { + t.Errorf("%s: size %d, want at least %d", name(i), len(got), offset1+len(data1)) + lost++ + continue + } + ok0 := bytes.Equal(got[offset0:offset0+len(data0)], data0) + ok1 := bytes.Equal(got[offset1:offset1+len(data1)], data1) + if !ok0 || !ok1 { + t.Errorf("%s: node 0 data intact=%v, node 1 data intact=%v", name(i), ok0, ok1) + lost++ + } + } + if lost > 0 { + t.Fatalf("%d of %d files lost a node's write", lost, files) + } +} diff --git a/tests/integration/tree_reservation_test.go b/tests/integration/tree_reservation_test.go new file mode 100644 index 00000000..5e8a3498 --- /dev/null +++ b/tests/integration/tree_reservation_test.go @@ -0,0 +1,87 @@ +package integration + +import ( + "syscall" + "testing" +) + +// TestTreeReservationsDoNotLeakBlocks appends to a file whose two inline extents +// are full, so each append reserves an extent-tree block and then merges into the +// last extent without using it. Unused reservations must go back to the bitmap, +// not to an in-memory list that unmount drops, so the whole volume is writable +// again once the file is deleted. +func TestTreeReservationsDoNotLeakBlocks(t *testing.T) { + mfs := NewMountedFS(t, 64) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + // 128KB at block 0 keeps the file off the SFB path; block 2 makes the second extent. + if err := mfs.WriteFileAt("append.bin", chunk(1, 128*1024), 0); err != nil { + t.Fatalf("write at block 0 failed: %v", err) + } + for blk := 2; blk <= 7; blk++ { + if err := mfs.WriteFileAt("append.bin", chunk(byte(blk), 64*1024), int64(blk*mb)); err != nil { + t.Fatalf("write at block %d failed: %v", blk, err) + } + } + remount(t, mfs) + if err := mfs.Remove("append.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + + var st syscall.Statfs_t + if err := syscall.Statfs(mfs.MountPoint, &st); err != nil { + t.Fatalf("Statfs failed: %v", err) + } + // Every data block except the root directory's. + fill := int(st.Blocks-1) * mb + if err := mfs.WriteFile("fill.bin", make([]byte, fill)); err != nil { + t.Fatalf("writing %d MB into an empty volume of %d blocks failed: %v", fill/mb, st.Blocks, err) + } +} + +// TestWriteInsideFallocatedRange writes 4KB into the middle of a fallocated +// range. Clearing the unwritten flag splits one extent into three, which needs +// extent-tree blocks, and the rest of the block must read as zeros even though +// the block held another file's data before. +func TestWriteInsideFallocatedRange(t *testing.T) { + mfs := NewMountedFS(t, 64) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + old := make([]byte, 4*mb) + for i := range old { + old[i] = 0xEE + } + if err := mfs.WriteFile("old.bin", old); err != nil { + t.Fatalf("write old.bin failed: %v", err) + } + if err := mfs.Remove("old.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + + if err := mfs.WriteFile("falloc.bin", nil); err != nil { + t.Fatalf("create falloc.bin failed: %v", err) + } + f, err := mfs.OpenFile("falloc.bin") + if err != nil { + t.Fatalf("OpenFile failed: %v", err) + } + if err := syscall.Fallocate(int(f.Fd()), 0, 0, 4*mb); err != nil { + f.Close() + t.Fatalf("Fallocate failed: %v", err) + } + data := chunk(7, 4096) + const at = mb + 512*1024 + if _, err := f.WriteAt(data, at); err != nil { + f.Close() + t.Fatalf("write inside the fallocated range failed: %v", err) + } + f.Close() + + remount(t, mfs) + want := make([]byte, 4*mb) + copy(want[at:], data) + checkFile(t, mfs, "falloc.bin", want) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index ebd2a181..1dc2e029 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -304,6 +304,35 @@ scenarios: docker: run-tests.sh::test_mixed_size_writes vm: run-vm-tests.sh::test_mixed_size_writes + - id: multinode-concurrent-first-write-same-block + name: Concurrent first write to the same block + description: Two nodes write different ranges of a new file's first block at once; both writes survive + priority: high + environments: + go: multinode_test.go::TestMultiNodeConcurrentFirstWriteToSameBlock + docker: run-tests.sh::test_concurrent_first_write_same_block + vm: run-vm-tests.sh::test_concurrent_first_write_same_block + + # Docker/VM TODO: single-node behavior that needs a remount and a full-volume + # write; the Go integration test runs the real mount binary already. + - id: tree-reservation-no-leak + name: Extent-tree reservations do not leak blocks + description: Appends that merge into a full inline extent return their unused tree-block reservation, so the whole volume is writable after the file is deleted and the FS remounted + priority: medium + environments: + go: tree_reservation_test.go::TestTreeReservationsDoNotLeakBlocks + docker: TODO + vm: TODO + + - id: write-inside-fallocated-range + name: Write inside a fallocated range + description: A 4KB write into the middle of a fallocated range splits the unwritten extent under the live lock and zero-fills the rest of the block + priority: medium + environments: + go: tree_reservation_test.go::TestWriteInsideFallocatedRange + docker: TODO + vm: TODO + - id: multinode-overlapping-block-writes name: Overlapping block writes description: Multiple nodes write to same block at different offsets @@ -1400,3 +1429,126 @@ scenarios: priority: go_only environments: go: internal/journal/journal_fuzz_test.go::FuzzJournalScan + + # =========================================================================== + # DISTRIBUTED LOCK SAFETY (internal/lock) + # Two or three lock managers share one device to act as separate nodes. + # =========================================================================== + + # Docker/VM TODO: a shell test would have to freeze a node (docker pause, or + # SIGSTOP on the QEMU process) inside a lock-held section that lasts + # milliseconds, then resume it after StaleThreshold. Shell tests cannot target + # that window, so the Go test simulates the pause by aging the on-disk record. + - id: lock-stale-reclaim-paused-holder + name: Stale lock reclaimed from paused holder + description: A holder stalled past StaleThreshold loses its lock to another node without being told + priority: high + environments: + go: ../../internal/lock/safety_test.go::TestStaleLockIsReclaimedFromPausedHolder + docker: TODO + vm: TODO + + - id: lock-paused-holder-release + name: Paused holder release keeps new holder's lock + description: A late Release from a holder whose lock was reclaimed returns ErrNotLockOwner and leaves the new holder's record intact + priority: go_only + environments: + go: internal/lock/safety_test.go::TestPausedHolderReleaseKeepsReclaimedLock + + - id: lock-colliding-resources-excluded + name: Colliding resources exclude other nodes + description: A held record blocks other nodes from every resource that hashes to it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestCollidingResourceIsBlockedWhileOtherNodeHoldsRecord + + - id: lock-same-node-colliding-resources + name: One node holds colliding resources + description: A node can hold two resources that share a record; the record stays held until the last one is released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNodeHoldsCollidingResourcesUntilLastRelease + + - id: lock-same-node-concurrent-acquire + name: Concurrent same-node acquire keeps the lock + description: Goroutines on one node acquiring the same lock at once leave it held on disk + priority: go_only + environments: + go: internal/lock/safety_test.go::TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut + + - id: lock-shared-release-by-count + name: Shared lock survives another reader's release + description: Releasing one reader's shared hold does not free the lock for writers while other readers hold it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSharedLockHeldAfterAnotherReaderReleases + + - id: lock-exclusive-waits-same-node-readers + name: Exclusive waits for same-node readers + description: An exclusive request does not join a shared hold from the same node; it waits until this node's readers release + priority: go_only + environments: + go: internal/lock/safety_test.go::TestExclusiveWaitsForSameNodeReaders + + - id: lock-upgrade-same-goroutine + name: Shared to exclusive upgrade rejected + description: A goroutine holding a shared lock gets ErrDeadlockRisk when it asks for exclusive on the same record + priority: go_only + environments: + go: internal/lock/safety_test.go::TestSameGoroutineCannotUpgradeSharedToExclusive + + - id: lock-two-node-readers-cannot-write + name: Readers on two nodes cannot both write + description: When two nodes hold a record shared, neither node's exclusive request succeeds + priority: go_only + environments: + go: internal/lock/safety_test.go::TestReadersOnTwoNodesCannotBothWrite + + - id: lock-join-fails-after-takeover + name: Join fails after takeover + description: A key joining an exclusive hold that another node took over as stale gets ErrLockLost instead of joining + priority: go_only + environments: + go: internal/lock/safety_test.go::TestJoinFailsAfterRecordTakenOver + + - id: lock-busy-record-renewed + name: Busy record renewed on join + description: A key joining an exclusive record this node has held past renewAfter rewrites its AcquireTime, so a busy node does not lose the record as stale + priority: go_only + environments: + go: internal/lock/safety_test.go::TestJoiningKeyRenewsBusyRecord + + - id: inode-write-same-node-reader + name: Inode write does not hang behind same-node reader + description: Table.Write on a directory finishes while another goroutine on the node holds the directory's shared lock and reads the inode table + priority: go_only + environments: + go: internal/inode/lock_test.go::TestWriteDoesNotHangBehindSameNodeReader + + - id: lock-stale-shared-reclaim + name: Stale shared lock reclaim drops dead readers + description: Reclaiming a stale shared record resets the reader count so it can be fully released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestStaleSharedLockReclaimResetsReaderCount + + - id: lock-leaked-shared-count-goes-stale + name: Leaked reader count goes stale + description: New readers do not refresh a shared record's age, so a reader count leaked by a lost update cannot block writers forever + priority: go_only + environments: + go: internal/lock/safety_test.go::TestNewReadersDoNotKeepLeakedSharedLockFresh + + - id: write-failure-keeps-committed-blocks + name: Failed write keeps blocks committed mid-write + description: When the final inode write fails after a mid-write persist committed a new block, the retry finds that block still allocated and mapped + priority: go_only + environments: + go: internal/goclusterfs/lock_commit_test.go::TestFailedWriteKeepsBlocksCommittedMidWrite + + - id: lock-region-bounded + name: Lock region stays bounded + description: Many acquire/release cycles across resources and two nodes reuse the fixed region, leave no record held, and write nothing outside it + priority: go_only + environments: + go: internal/lock/safety_test.go::TestLockRegionStaysBoundedAcrossCycles diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 08863be3..25df7901 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2410,6 +2410,48 @@ test_stability() { # Concurrent Same-File Write Tests (using pre-extension pattern) # ============================================================================= +# Two VMs write different ranges of each new file's first block at once. +# Without the inode lock the later inode write drops the other VM's extent. +test_concurrent_first_write_same_block() { + log_info "Testing concurrent first writes to the same block (2 VMs)..." + ensure_cluster_healthy 2 + + local files=10 + local size=65536 + local offset0=262144 + local offset1=524288 + + for i in $(seq 1 $files); do + ssh_run 1 "touch /mnt/goclusterfs/first_write_$i.bin" + done + sleep 2 + + for i in $(seq 1 $files); do + write_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$size" "$offset0" & + write_pattern_at_on_vm 2 "first_write_$i.bin" "B" "$size" "$offset1" & + wait + done + sleep 2 + + # Restart the reader so cached extents from earlier files with the same inode numbers cannot hide the result. + stop_vm 1 + start_vm 1 false + wait_vm_healthy 1 || return 1 + setup_goclusterfs 1 false + + local lost=0 + for i in $(seq 1 $files); do + verify_pattern_at_on_vm 1 "first_write_$i.bin" "A" "$offset0" 4096 || lost=$((lost + 1)) + verify_pattern_at_on_vm 1 "first_write_$i.bin" "B" "$offset1" 4096 || lost=$((lost + 1)) + done + if [[ $lost -gt 0 ]]; then + log_error "$lost of $((files * 2)) concurrent first writes lost" + return 1 + fi + log_success "All concurrent first writes survived" + return 0 +} + # Test 10: Two VMs write to same file at different offsets test_concurrent_same_file_writes() { log_info "Testing concurrent same-file writes (2 VMs)..." @@ -2956,6 +2998,7 @@ main() { run_test "Long-Running Stability" test_stability # Run concurrent same-file write tests (using pre-extension pattern) + run_test "Concurrent First Write Same Block" test_concurrent_first_write_same_block run_test "Concurrent Same-File Writes (2 VMs)" test_concurrent_same_file_writes run_test "Concurrent Same-File Writes (3 VMs)" test_concurrent_same_file_writes_3vms run_test "Mixed Size Writes" test_mixed_size_writes