diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 87e5c32d..0fc3a4c0 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -332,6 +332,31 @@ func (f *FS) updateInode(ctx context.Context, num uint32, update func(*inode.Ino } } +// updateInodeLocked is updateInode for changes that touch no extents or blocks, +// such as times and link counts. It holds the inode's lock from the read to the +// write, so it cannot write back an inode that another node changed in between. +func (f *FS) updateInodeLocked(ctx context.Context, num uint32, update func(*inode.Inode) error) (*inode.Inode, error) { + defer f.lockInode(num)() + if f.lockMgr != nil { + if err := f.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(num), lock.LockModeExclusive); err != nil { + return nil, fmt.Errorf("failed to acquire inode lock: %w", err) + } + defer f.lockMgr.Release(lock.LockTypeInode, uint64(num)) + } + + ino, err := f.inodeTbl.Read(num) + if err != nil { + return nil, err + } + if err := update(ino); err != nil { + return nil, err + } + if err := f.inodeTbl.WriteWithLockHeld(ino); err != nil { + return nil, err + } + return ino, nil +} + // invalidateInode removes an inode from the cache before modification. // Safe to call even if cache is nil. func (f *FS) invalidateInode(inodeNum uint32) { @@ -719,7 +744,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, // dirMgr.Add() may have allocated a new block and updated extent fields. // Only Size is copied back to dirIno; extent fields are stale. // Writing the stale dirIno would overwrite the correct extent data on disk. - dirIno, err = f.updateInode(ctx, inode.InodeRoot, func(ino *inode.Inode) error { ino.Mtime = now; return nil }) + dirIno, err = f.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error { ino.Mtime = now; return nil }) if err != nil { log.Printf("[DEBUG] Create %q: updating root inode failed: %v", name, err) // The file was created successfully, just log the error @@ -876,13 +901,13 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f // dirMgr.Add() may have allocated a new block and updated extent fields. // Only Size is copied back to dirIno; extent fields are stale. // Writing the stale dirIno would overwrite the correct extent data on disk. - dirIno, err = f.updateInode(ctx, inode.InodeRoot, func(ino *inode.Inode) error { + dirIno, err = f.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error { ino.LinkCount++ ino.Mtime = now return nil }) if err != nil { - log.Printf("[DEBUG] Mkdir %q: updating root inode failed: %v", name, err) + log.Printf("[WARN] Mkdir %q: link count of directory inode %d not updated: %v", name, inode.InodeRoot, err) // The child directory was created successfully, just log the error } else { // Broadcast invalidation so other nodes refresh their directory cache @@ -987,15 +1012,15 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno { if sfbBacked { f.freeSFBSlot(ctx, sfbSlot) } - } else if _, err := f.updateInode(ctx, childIno.InodeNum, func(ino *inode.Inode) error { + } else if _, err := f.updateInodeLocked(ctx, childIno.InodeNum, func(ino *inode.Inode) error { ino.LinkCount-- ino.Ctime = now return nil }); err != nil { - log.Printf("[DEBUG] Unlink %q: updating link count failed: %v", name, err) + log.Printf("[WARN] Unlink %q: link count of inode %d not updated: %v", name, childIno.InodeNum, err) } - if _, err := f.updateInode(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }); err != nil { + if _, err := f.updateInodeLocked(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }); err != nil { log.Printf("[DEBUG] Unlink %q: updating directory failed: %v", name, err) } @@ -1072,12 +1097,12 @@ func (f *FS) Rmdir(ctx context.Context, name string) syscall.Errno { f.inodeTbl.Free(ctx, childIno.InodeNum) now := time.Now().UnixNano() - if _, err := f.updateInode(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { + if _, err := f.updateInodeLocked(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { ino.LinkCount-- ino.Mtime = now return nil }); err != nil { - log.Printf("[DEBUG] Rmdir %q: updating directory failed: %v", name, err) + log.Printf("[WARN] Rmdir %q: link count of directory inode %d not updated: %v", name, dirIno.InodeNum, err) } // === WAL: Mark intent complete === @@ -1423,7 +1448,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod // dirMgr.Add() may have allocated a new block and updated extent fields. // Only Size is copied back to dirIno; extent fields are stale. // Writing the stale dirIno would overwrite the correct extent data on disk. - dirIno, err = d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }) + dirIno, err = d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }) if err != nil { log.Printf("[DEBUG] Create %q: updating parent inode %d failed: %v", name, d.inodeNum, err) // The file was created successfully, just log the error @@ -1640,14 +1665,14 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out * // dirMgr.Add() may have allocated a new block and updated extent fields. // Only Size is copied back to dirIno; extent fields are stale. // Writing the stale dirIno would overwrite the correct extent data on disk. - dirIno, err = d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error { + dirIno, err = d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error { // Increment parent link count (for ..) ino.LinkCount++ ino.Mtime = now return nil }) if err != nil { - log.Printf("[DEBUG] Mkdir %q: updating parent inode %d failed: %v", name, d.inodeNum, err) + log.Printf("[WARN] Mkdir %q: link count of directory inode %d not updated: %v", name, d.inodeNum, err) // The child directory was created successfully, just log the error // and skip updating parent LinkCount/Mtime } else { @@ -1857,8 +1882,8 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall. if sfbBacked { d.fs.freeSFBSlot(ctx, sfbSlot) } - } else if updated, err := d.fs.updateInode(ctx, uint32(childInodeNum), func(ino *inode.Inode) error { ino.LinkCount--; return nil }); err != nil { - log.Printf("[DEBUG] removeEntry %q: updating link count failed: %v", name, err) + } else if updated, err := d.fs.updateInodeLocked(ctx, uint32(childInodeNum), func(ino *inode.Inode) error { ino.LinkCount--; return nil }); err != nil { + log.Printf("[WARN] removeEntry %q: link count of inode %d not updated: %v", name, childInodeNum, err) } else { // Cache the updated child inode d.fs.cacheInode(updated) @@ -1866,14 +1891,14 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall. // Update directory mtime, and its link count for directory removal now := time.Now().UnixNano() - if updated, err := d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error { + if updated, err := d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error { if isDir { ino.LinkCount-- } ino.Mtime = now return nil }); err != nil { - log.Printf("[DEBUG] removeEntry %q: updating directory failed: %v", name, err) + log.Printf("[WARN] removeEntry %q: directory inode %d not updated: %v", name, d.inodeNum, err) } else { // Cache the updated parent directory inode d.fs.cacheInode(updated) diff --git a/internal/goclusterfs/inode_update_test.go b/internal/goclusterfs/inode_update_test.go new file mode 100644 index 00000000..ed84604d --- /dev/null +++ b/internal/goclusterfs/inode_update_test.go @@ -0,0 +1,50 @@ +package goclusterfs + +import ( + "context" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/inode" +) + +// TestUpdateInodeLockedWaitsForOtherNode runs another node's link count update +// between this node's read and write of the root directory inode. Neither +// update may be lost. +func TestUpdateInodeLockedWaitsForOtherNode(t *testing.T) { + path := newFormattedImage(t) + nodeA, nodeB := openFormattedFS(t, path), openFormattedFS(t, path) + ctx := context.Background() + increment := func(ino *inode.Inode) error { ino.LinkCount++; return nil } + + before, err := nodeA.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + + otherDone := make(chan error, 1) + _, err = nodeA.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error { + go func() { + _, err := nodeB.updateInodeLocked(ctx, inode.InodeRoot, increment) + otherDone <- err + }() + // Without the lock, node B reads and writes the inode in this window. + time.Sleep(200 * time.Millisecond) + return increment(ino) + }) + if err != nil { + t.Fatalf("node A update: %v", err) + } + if err := <-otherDone; err != nil { + t.Fatalf("node B update: %v", err) + } + + after, err := nodeA.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + if after.LinkCount != before.LinkCount+2 { + t.Errorf("root link count %d after two increments from %d, want %d", + after.LinkCount, before.LinkCount, before.LinkCount+2) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 72b6c95f..bc82fec4 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1355,6 +1355,13 @@ scenarios: environments: go: internal/cache/extent_cache_test.go::TestGetExtentCountsStaleEntryAsMiss + - id: inode-update-locked-two-nodes + name: Locked inode update waits for another node + description: Another node's link count update between this node's read and write of a directory inode is not lost + priority: go_only + environments: + go: internal/goclusterfs/inode_update_test.go::TestUpdateInodeLockedWaitsForOtherNode + - id: broadcast-ignore-own name: Broadcast ignores own messages description: Broadcaster ignores messages from itself