Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/cache/inode_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func (m *Manager) cacheInode(in *inode.Inode) {
LastValidated: now,
}
m.inodes.Put(in.InodeNum, cached)
m.negatives.Delete(in.InodeNum)
}

// updateValidationTime updates the LastValidated timestamp for a cached inode.
Expand Down
110 changes: 110 additions & 0 deletions internal/cache/inode_cache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package cache

import (
"context"
"os"
"testing"
"time"

"github.com/supporttools/GoClusterFS/internal/inode"
"github.com/supporttools/GoClusterFS/internal/storage"
)

const testTableSize = 64 * inode.SectorSize

func newTestInodeTable(t *testing.T) *inode.Table {
t.Helper()

dir := os.Getenv("GOCLUSTERFS_TEST_DIR")
if dir == "" {
dir = "."
}
f, err := os.CreateTemp(dir, "test_cache_inodes_*.bin")
if err != nil {
t.Skipf("Cannot create test file: %v", err)
}
path := f.Name()
t.Cleanup(func() { os.Remove(path) })
err = f.Truncate(testTableSize)
f.Close()
if err != nil {
t.Skipf("Cannot truncate test file: %v", err)
}

dev, err := storage.Open(path)
if err != nil {
t.Skipf("O_DIRECT not supported: %v", err)
}
t.Cleanup(func() { dev.Close() })

return inode.NewTable(dev, 0, testTableSize, nil, [16]byte{})
}

func claimWithSize(t *testing.T, tbl *inode.Table, num uint32, size uint64) *inode.Inode {
t.Helper()
ctx := context.Background()
ino, err := tbl.ClaimSlot(ctx, num)
if err != nil {
t.Fatalf("ClaimSlot(%d): %v", num, err)
}
ino.Size = size
if err := tbl.Write(ctx, ino); err != nil {
t.Fatalf("Write(%d): %v", num, err)
}
return ino
}

func TestGetInodeDetectsSlotReuse(t *testing.T) {
tbl := newTestInodeTable(t)
cfg := DefaultConfig()
cfg.InodeTTL = time.Millisecond
mgr, err := NewManager(cfg, tbl, nil)
if err != nil {
t.Fatal(err)
}

old := claimWithSize(t, tbl, inode.InodeFirstUsable, 3)
if _, err := mgr.GetInode(old.InodeNum); err != nil {
t.Fatalf("GetInode: %v", err)
}

if err := tbl.Free(context.Background(), old.InodeNum); err != nil {
t.Fatalf("Free: %v", err)
}
reused := claimWithSize(t, tbl, old.InodeNum, 10)
if reused.DataGeneration <= old.DataGeneration || reused.MetaGeneration <= old.MetaGeneration {
t.Errorf("reused generations (%d, %d) not above old (%d, %d)",
reused.DataGeneration, reused.MetaGeneration, old.DataGeneration, old.MetaGeneration)
}

time.Sleep(2 * cfg.InodeTTL)
got, err := mgr.GetInode(old.InodeNum)
if err != nil {
t.Fatalf("GetInode after reuse: %v", err)
}
if got.Size != 10 {
t.Errorf("GetInode after reuse returned Size %d, want 10", got.Size)
}
}

func TestCacheInodeClearsNegativeEntry(t *testing.T) {
tbl := newTestInodeTable(t)
mgr, err := NewManager(DefaultConfig(), tbl, nil)
if err != nil {
t.Fatal(err)
}

if _, err := mgr.GetInode(inode.InodeFirstUsable); err != inode.ErrInodeNotInUse {
t.Fatalf("GetInode on free slot: err=%v, want ErrInodeNotInUse", err)
}

mgr.CacheInode(claimWithSize(t, tbl, inode.InodeFirstUsable, 5))

got, err := mgr.GetInode(inode.InodeFirstUsable)
if err != nil {
t.Fatalf("GetInode after CacheInode: %v", err)
}
if got.Size != 5 {
t.Errorf("GetInode returned Size %d, want 5", got.Size)
}
}
2 changes: 2 additions & 0 deletions internal/goclusterfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -1840,6 +1840,8 @@ func (f *File) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, s
return nil, 0, syscall.EISDIR
}
log.Printf("[DEBUG] File.Open: success, Size=%d", ino.Size)
// Close-to-open consistency: reads through this handle start from the on-disk inode.
f.fs.cacheInode(ino)
// Return nil FileHandle - the File node itself implements Read/Write/etc.
// The fuse.FOPEN_KEEP_CACHE flag tells the kernel to keep cached data valid
// (we don't use it in cluster mode since other nodes may modify the file).
Expand Down
43 changes: 23 additions & 20 deletions internal/inode/inode.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,21 +260,27 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) {
return nil, ErrInodeInUse
}

// Initialize the inode
newInode := &Inode{
Magic: InodeMagic,
InodeNum: num,
LinkCount: 1,
LastModifierUUID: t.nodeUUID,
}

newInode := t.newInodeInSlot(num, existing)
if err := t.writeInodeUnlocked(newInode); err != nil {
return nil, fmt.Errorf("failed to write new inode: %w", err)
}

return newInode, nil
}

// newInodeInSlot initializes an inode for a free slot. Its generations are higher
// than any previous occupant's, so caches on other nodes can detect the reuse.
func (t *Table) newInodeInSlot(num uint32, free *Inode) *Inode {
return &Inode{
Magic: InodeMagic,
InodeNum: num,
LinkCount: 1,
DataGeneration: free.DataGeneration + 1,
MetaGeneration: free.MetaGeneration + 1,
LastModifierUUID: t.nodeUUID,
}
}

// Alloc allocates a new inode.
// Returns the allocated inode with InodeNum set, ready for initialization.
//
Expand All @@ -293,13 +299,7 @@ func (t *Table) Alloc(ctx context.Context) (*Inode, error) {

// Check if inode is free (Magic == 0)
if inode.Magic == 0 {
// Found a free slot - initialize it
newInode := &Inode{
Magic: InodeMagic,
InodeNum: num,
LinkCount: 1,
LastModifierUUID: t.nodeUUID,
}
newInode := t.newInodeInSlot(num, inode)

// Acquire distributed lock if available
if t.lockMgr != nil {
Expand Down Expand Up @@ -352,9 +352,11 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error {
return ErrInodeNotInUse
}

// Clear the inode (set Magic to 0)
// Keep the generations so the slot's next occupant gets higher ones.
emptyInode := &Inode{
InodeNum: inodeNum,
InodeNum: inodeNum,
DataGeneration: inode.DataGeneration,
MetaGeneration: inode.MetaGeneration,
}

return t.writeInodeUnlocked(emptyInode)
Expand Down Expand Up @@ -567,11 +569,12 @@ func deserializeInode(buf []byte) (*Inode, error) {
return nil, ErrInvalidInode
}

// Check for empty inode (all zeros = free)
magic := binary.LittleEndian.Uint32(buf[0:4])
if magic == 0 {
// Return empty inode with just the position
return &Inode{}, nil
return &Inode{
DataGeneration: binary.LittleEndian.Uint64(buf[72:80]),
MetaGeneration: binary.LittleEndian.Uint64(buf[80:88]),
}, nil
}

// Verify checksum
Expand Down
4 changes: 4 additions & 0 deletions internal/journal/cross_node_recovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,10 @@ func applyInodeCreate(ctx context.Context, rctx *RecoveryContext, payload []byte
GID: p.GID,
LinkCount: 1,
}
if err == nil {
ino.DataGeneration = existing.DataGeneration + 1
ino.MetaGeneration = existing.MetaGeneration + 1
}

now := time.Now().UnixNano()
ino.Atime = now
Expand Down
43 changes: 43 additions & 0 deletions tests/docker/scripts/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2171,6 +2171,48 @@ test_broadcast_bidirectional() {
return 0
}

# Test: Read after cross-node inode reuse
# Node 0 deletes a file node 1 has read and creates a new file with the same inode number.
# Node 1 must read the new file's full content, not the old size or old data block.
test_inode_reuse_stale_read() {
log "Testing read after cross-node inode reuse..."

local writer="${NODE_ARRAY[0]}"
local reader="${NODE_ARRAY[1]}"
local dir=/mnt/goclusterfs
local old="reuse_old_$(date +%s).bin"

exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' o > $dir/$old"
local old_ino
old_ino=$(exec_on_node "$writer" stat -c %i "$dir/$old")
exec_on_node "$reader" sh -c "cat $dir/$old > /dev/null"

# New files take the lowest free inode, so create files until one gets the old number.
local new
new=$(exec_on_node "$writer" sh -c "rm $dir/$old; for i in \$(seq 1 200); do f=reuse_new_\$i.bin; touch $dir/\$f; if [ \$(stat -c %i $dir/\$f) = $old_ino ]; then echo \$f; break; fi; done")
if [[ -z "$new" ]]; then
log "ERROR: No new file reused inode $old_ino"
exec_on_node "$writer" sh -c "rm -f $dir/reuse_new_*.bin"
return 1
fi

exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' p > $dir/reuse_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$new"
# The healthcheck's ls can leave a pre-write size in the reader's 1s kernel attribute cache.
sleep 1.5
local want got
want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$new")
got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$new")
exec_on_node "$writer" sh -c "rm -f $dir/reuse_new_*.bin $dir/reuse_pad.bin"

if [[ "$got" != "$want" ]]; then
log "ERROR: $reader read wrong data from $new (inode $old_ino): md5 '$got', want '$want'"
return 1
fi

log "Inode reuse stale read test passed"
return 0
}

# Test 4: Rolling restart
test_rolling_restart() {
log "Testing rolling restart..."
Expand Down Expand Up @@ -2576,6 +2618,7 @@ run_test "Broadcast Invalidation" test_broadcast_invalidation
run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidation
run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates
run_test "Broadcast Bidirectional" test_broadcast_bidirectional
run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read
run_test "Rolling Restart" test_rolling_restart
run_test "Stress Random Failures" test_stress_random_failures

Expand Down
2 changes: 2 additions & 0 deletions tests/integration/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
type MountedFS struct {
ImagePath string
MountPoint string
ExtraArgs []string // Appended to the mount command by Mount
MountCmd *exec.Cmd
BinaryPath string
t *testing.T
Expand Down Expand Up @@ -190,6 +191,7 @@ func (m *MountedFS) Mount(debug bool) error {
if debug {
args = append(args, "-debug")
}
args = append(args, m.ExtraArgs...)

// Create a fresh context for this mount operation
m.ctx, m.cancel = context.WithCancel(context.Background())
Expand Down
80 changes: 80 additions & 0 deletions tests/integration/multinode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"math/rand"
"sync"
"syscall"
"testing"
"time"
)
Expand Down Expand Up @@ -1054,6 +1055,85 @@ func TestMultiNodeReversedWriteOrder(t *testing.T) {
t.Logf("Reversed write order test completed - file size: %d bytes", len(fullContent))
}

// TestMultiNodeInodeReuseStaleRead checks that a node which read a file reads the
// right data after another node deletes it and a new file reuses its inode number.
func TestMultiNodeInodeReuseStaleRead(t *testing.T) {
cluster := NewMultiNodeCluster(t, 2, 50)
if err := cluster.FormatAndStartFirst(false); err != nil {
t.Fatalf("Failed to format and start first node: %v", err)
}
// The reader must rely on its own cache validation, like VMs on separate networks.
cluster.Node(1).ExtraArgs = []string{"-broadcast=false"}
if err := cluster.StartNode(1, false); err != nil {
t.Fatalf("Failed to start second node: %v", err)
}
writer, reader := cluster.Node(0), cluster.Node(1)

t.Run("SmallFile", func(t *testing.T) {
readAfterInodeReuse(t, writer, reader, "small",
[]byte("v5\n"), []byte("from vm 1\n"), nil)
})

// The padding file pushes the new data away from the old block, so a stale extent mapping reads wrong data.
t.Run("BlockFile", func(t *testing.T) {
readAfterInodeReuse(t, writer, reader, "block",
bytes.Repeat([]byte("o"), 200<<10),
bytes.Repeat([]byte("n"), 300<<10),
bytes.Repeat([]byte("p"), 200<<10))
})
}

func readAfterInodeReuse(t *testing.T, writer, reader *MountedFS, prefix string, oldData, newData, padData []byte) {
t.Helper()
oldName, newName, padName := prefix+"_old", prefix+"_new", prefix+"_pad"

if err := writer.WriteFile(oldName, oldData); err != nil {
t.Fatalf("Writer: create %s: %v", oldName, err)
}
oldIno := inodeNumber(t, writer, oldName)

got, err := reader.ReadFile(oldName)
if err != nil || !bytes.Equal(got, oldData) {
t.Fatalf("Reader: read %s: %d bytes, err=%v, want %d bytes", oldName, len(got), err, len(oldData))
}

if err := writer.Remove(oldName); err != nil {
t.Fatalf("Writer: remove %s: %v", oldName, err)
}
if err := writer.WriteFile(newName, nil); err != nil {
t.Fatalf("Writer: create %s: %v", newName, err)
}
if ino := inodeNumber(t, writer, newName); ino != oldIno {
t.Fatalf("%s got inode %d, want reused inode %d", newName, ino, oldIno)
}
if padData != nil {
if err := writer.WriteFile(padName, padData); err != nil {
t.Fatalf("Writer: create %s: %v", padName, err)
}
}
if err := writer.WriteFile(newName, newData); err != nil {
t.Fatalf("Writer: write %s: %v", newName, err)
}

got, err = reader.ReadFile(newName)
if err != nil {
t.Fatalf("Reader: read %s: %v", newName, err)
}
if !bytes.Equal(got, newData) {
t.Errorf("Reader: %s has %d bytes starting %q, want %d bytes starting %q",
newName, len(got), got[:min(len(got), 16)], len(newData), newData[:min(len(newData), 16)])
}
}

func inodeNumber(t *testing.T, m *MountedFS, name string) uint64 {
t.Helper()
fi, err := m.Stat(name)
if err != nil {
t.Fatalf("stat %s: %v", name, err)
}
return fi.Sys().(*syscall.Stat_t).Ino
}

// TestNetworkPartition simulates network partition handling by testing node isolation
// and reconnection. Since Go integration tests run multiple FUSE mount processes on
// the same host (sharing a backing image), true network partition cannot be simulated.
Expand Down
Loading
Loading