Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 40 additions & 15 deletions internal/goclusterfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,31 @@ func (f *FS) updateInode(ctx context.Context, num uint32, update func(*inode.Ino
}
}

// updateInodeLocked is updateInode for changes that touch no extents or blocks,
// such as times and link counts. It holds the inode's lock from the read to the
// write, so it cannot write back an inode that another node changed in between.
func (f *FS) updateInodeLocked(ctx context.Context, num uint32, update func(*inode.Inode) error) (*inode.Inode, error) {
defer f.lockInode(num)()
if f.lockMgr != nil {
if err := f.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(num), lock.LockModeExclusive); err != nil {
return nil, fmt.Errorf("failed to acquire inode lock: %w", err)
}
defer f.lockMgr.Release(lock.LockTypeInode, uint64(num))
}

ino, err := f.inodeTbl.Read(num)
if err != nil {
return nil, err
}
if err := update(ino); err != nil {
return nil, err
}
if err := f.inodeTbl.WriteWithLockHeld(ino); err != nil {
return nil, err
}
return ino, nil
}

// invalidateInode removes an inode from the cache before modification.
// Safe to call even if cache is nil.
func (f *FS) invalidateInode(inodeNum uint32) {
Expand Down Expand Up @@ -719,7 +744,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32,
// dirMgr.Add() may have allocated a new block and updated extent fields.
// Only Size is copied back to dirIno; extent fields are stale.
// Writing the stale dirIno would overwrite the correct extent data on disk.
dirIno, err = f.updateInode(ctx, inode.InodeRoot, func(ino *inode.Inode) error { ino.Mtime = now; return nil })
dirIno, err = f.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error { ino.Mtime = now; return nil })
if err != nil {
log.Printf("[DEBUG] Create %q: updating root inode failed: %v", name, err)
// The file was created successfully, just log the error
Expand Down Expand Up @@ -876,13 +901,13 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f
// dirMgr.Add() may have allocated a new block and updated extent fields.
// Only Size is copied back to dirIno; extent fields are stale.
// Writing the stale dirIno would overwrite the correct extent data on disk.
dirIno, err = f.updateInode(ctx, inode.InodeRoot, func(ino *inode.Inode) error {
dirIno, err = f.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error {
ino.LinkCount++
ino.Mtime = now
return nil
})
if err != nil {
log.Printf("[DEBUG] Mkdir %q: updating root inode failed: %v", name, err)
log.Printf("[WARN] Mkdir %q: link count of directory inode %d not updated: %v", name, inode.InodeRoot, err)
// The child directory was created successfully, just log the error
} else {
// Broadcast invalidation so other nodes refresh their directory cache
Expand Down Expand Up @@ -987,15 +1012,15 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno {
if sfbBacked {
f.freeSFBSlot(ctx, sfbSlot)
}
} else if _, err := f.updateInode(ctx, childIno.InodeNum, func(ino *inode.Inode) error {
} else if _, err := f.updateInodeLocked(ctx, childIno.InodeNum, func(ino *inode.Inode) error {
ino.LinkCount--
ino.Ctime = now
return nil
}); err != nil {
log.Printf("[DEBUG] Unlink %q: updating link count failed: %v", name, err)
log.Printf("[WARN] Unlink %q: link count of inode %d not updated: %v", name, childIno.InodeNum, err)
}

if _, err := f.updateInode(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }); err != nil {
if _, err := f.updateInodeLocked(ctx, dirIno.InodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil }); err != nil {
log.Printf("[DEBUG] Unlink %q: updating directory failed: %v", name, err)
}

Expand Down Expand Up @@ -1072,12 +1097,12 @@ func (f *FS) Rmdir(ctx context.Context, name string) syscall.Errno {
f.inodeTbl.Free(ctx, childIno.InodeNum)

now := time.Now().UnixNano()
if _, err := f.updateInode(ctx, dirIno.InodeNum, func(ino *inode.Inode) error {
if _, err := f.updateInodeLocked(ctx, dirIno.InodeNum, func(ino *inode.Inode) error {
ino.LinkCount--
ino.Mtime = now
return nil
}); err != nil {
log.Printf("[DEBUG] Rmdir %q: updating directory failed: %v", name, err)
log.Printf("[WARN] Rmdir %q: link count of directory inode %d not updated: %v", name, dirIno.InodeNum, err)
}

// === WAL: Mark intent complete ===
Expand Down Expand Up @@ -1423,7 +1448,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod
// dirMgr.Add() may have allocated a new block and updated extent fields.
// Only Size is copied back to dirIno; extent fields are stale.
// Writing the stale dirIno would overwrite the correct extent data on disk.
dirIno, err = d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil })
dirIno, err = d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error { ino.Mtime = now; return nil })
if err != nil {
log.Printf("[DEBUG] Create %q: updating parent inode %d failed: %v", name, d.inodeNum, err)
// The file was created successfully, just log the error
Expand Down Expand Up @@ -1640,14 +1665,14 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out *
// dirMgr.Add() may have allocated a new block and updated extent fields.
// Only Size is copied back to dirIno; extent fields are stale.
// Writing the stale dirIno would overwrite the correct extent data on disk.
dirIno, err = d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error {
dirIno, err = d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error {
// Increment parent link count (for ..)
ino.LinkCount++
ino.Mtime = now
return nil
})
if err != nil {
log.Printf("[DEBUG] Mkdir %q: updating parent inode %d failed: %v", name, d.inodeNum, err)
log.Printf("[WARN] Mkdir %q: link count of directory inode %d not updated: %v", name, d.inodeNum, err)
// The child directory was created successfully, just log the error
// and skip updating parent LinkCount/Mtime
} else {
Expand Down Expand Up @@ -1857,23 +1882,23 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall.
if sfbBacked {
d.fs.freeSFBSlot(ctx, sfbSlot)
}
} else if updated, err := d.fs.updateInode(ctx, uint32(childInodeNum), func(ino *inode.Inode) error { ino.LinkCount--; return nil }); err != nil {
log.Printf("[DEBUG] removeEntry %q: updating link count failed: %v", name, err)
} else if updated, err := d.fs.updateInodeLocked(ctx, uint32(childInodeNum), func(ino *inode.Inode) error { ino.LinkCount--; return nil }); err != nil {
log.Printf("[WARN] removeEntry %q: link count of inode %d not updated: %v", name, childInodeNum, err)
} else {
// Cache the updated child inode
d.fs.cacheInode(updated)
}

// Update directory mtime, and its link count for directory removal
now := time.Now().UnixNano()
if updated, err := d.fs.updateInode(ctx, d.inodeNum, func(ino *inode.Inode) error {
if updated, err := d.fs.updateInodeLocked(ctx, d.inodeNum, func(ino *inode.Inode) error {
if isDir {
ino.LinkCount--
}
ino.Mtime = now
return nil
}); err != nil {
log.Printf("[DEBUG] removeEntry %q: updating directory failed: %v", name, err)
log.Printf("[WARN] removeEntry %q: directory inode %d not updated: %v", name, d.inodeNum, err)
} else {
// Cache the updated parent directory inode
d.fs.cacheInode(updated)
Expand Down
50 changes: 50 additions & 0 deletions internal/goclusterfs/inode_update_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package goclusterfs

import (
"context"
"testing"
"time"

"github.com/supporttools/GoClusterFS/internal/inode"
)

// TestUpdateInodeLockedWaitsForOtherNode runs another node's link count update
// between this node's read and write of the root directory inode. Neither
// update may be lost.
func TestUpdateInodeLockedWaitsForOtherNode(t *testing.T) {
path := newFormattedImage(t)
nodeA, nodeB := openFormattedFS(t, path), openFormattedFS(t, path)
ctx := context.Background()
increment := func(ino *inode.Inode) error { ino.LinkCount++; return nil }

before, err := nodeA.inodeTbl.Read(inode.InodeRoot)
if err != nil {
t.Fatal(err)
}

otherDone := make(chan error, 1)
_, err = nodeA.updateInodeLocked(ctx, inode.InodeRoot, func(ino *inode.Inode) error {
go func() {
_, err := nodeB.updateInodeLocked(ctx, inode.InodeRoot, increment)
otherDone <- err
}()
// Without the lock, node B reads and writes the inode in this window.
time.Sleep(200 * time.Millisecond)
return increment(ino)
})
if err != nil {
t.Fatalf("node A update: %v", err)
}
if err := <-otherDone; err != nil {
t.Fatalf("node B update: %v", err)
}

after, err := nodeA.inodeTbl.Read(inode.InodeRoot)
if err != nil {
t.Fatal(err)
}
if after.LinkCount != before.LinkCount+2 {
t.Errorf("root link count %d after two increments from %d, want %d",
after.LinkCount, before.LinkCount, before.LinkCount+2)
}
}
7 changes: 7 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1355,6 +1355,13 @@ scenarios:
environments:
go: internal/cache/extent_cache_test.go::TestGetExtentCountsStaleEntryAsMiss

- id: inode-update-locked-two-nodes
name: Locked inode update waits for another node
description: Another node's link count update between this node's read and write of a directory inode is not lost
priority: go_only
environments:
go: internal/goclusterfs/inode_update_test.go::TestUpdateInodeLockedWaitsForOtherNode

- id: broadcast-ignore-own
name: Broadcast ignores own messages
description: Broadcaster ignores messages from itself
Expand Down
Loading