diff --git a/internal/cache/inode_cache.go b/internal/cache/inode_cache.go index d8cccd91..7d075d5c 100644 --- a/internal/cache/inode_cache.go +++ b/internal/cache/inode_cache.go @@ -117,6 +117,7 @@ func (m *Manager) cacheInode(in *inode.Inode) { LastValidated: now, } m.inodes.Put(in.InodeNum, cached) + m.negatives.Delete(in.InodeNum) } // updateValidationTime updates the LastValidated timestamp for a cached inode. diff --git a/internal/cache/inode_cache_test.go b/internal/cache/inode_cache_test.go new file mode 100644 index 00000000..7839851f --- /dev/null +++ b/internal/cache/inode_cache_test.go @@ -0,0 +1,110 @@ +package cache + +import ( + "context" + "os" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +const testTableSize = 64 * inode.SectorSize + +func newTestInodeTable(t *testing.T) *inode.Table { + t.Helper() + + dir := os.Getenv("GOCLUSTERFS_TEST_DIR") + if dir == "" { + dir = "." + } + f, err := os.CreateTemp(dir, "test_cache_inodes_*.bin") + if err != nil { + t.Skipf("Cannot create test file: %v", err) + } + path := f.Name() + t.Cleanup(func() { os.Remove(path) }) + err = f.Truncate(testTableSize) + f.Close() + if err != nil { + t.Skipf("Cannot truncate test file: %v", err) + } + + dev, err := storage.Open(path) + if err != nil { + t.Skipf("O_DIRECT not supported: %v", err) + } + t.Cleanup(func() { dev.Close() }) + + return inode.NewTable(dev, 0, testTableSize, nil, [16]byte{}) +} + +func claimWithSize(t *testing.T, tbl *inode.Table, num uint32, size uint64) *inode.Inode { + t.Helper() + ctx := context.Background() + ino, err := tbl.ClaimSlot(ctx, num) + if err != nil { + t.Fatalf("ClaimSlot(%d): %v", num, err) + } + ino.Size = size + if err := tbl.Write(ctx, ino); err != nil { + t.Fatalf("Write(%d): %v", num, err) + } + return ino +} + +func TestGetInodeDetectsSlotReuse(t *testing.T) { + tbl := newTestInodeTable(t) + cfg := DefaultConfig() + cfg.InodeTTL = time.Millisecond + mgr, err := NewManager(cfg, tbl, nil) + if err != nil { + t.Fatal(err) + } + + old := claimWithSize(t, tbl, inode.InodeFirstUsable, 3) + if _, err := mgr.GetInode(old.InodeNum); err != nil { + t.Fatalf("GetInode: %v", err) + } + + if err := tbl.Free(context.Background(), old.InodeNum); err != nil { + t.Fatalf("Free: %v", err) + } + reused := claimWithSize(t, tbl, old.InodeNum, 10) + if reused.DataGeneration <= old.DataGeneration || reused.MetaGeneration <= old.MetaGeneration { + t.Errorf("reused generations (%d, %d) not above old (%d, %d)", + reused.DataGeneration, reused.MetaGeneration, old.DataGeneration, old.MetaGeneration) + } + + time.Sleep(2 * cfg.InodeTTL) + got, err := mgr.GetInode(old.InodeNum) + if err != nil { + t.Fatalf("GetInode after reuse: %v", err) + } + if got.Size != 10 { + t.Errorf("GetInode after reuse returned Size %d, want 10", got.Size) + } +} + +func TestCacheInodeClearsNegativeEntry(t *testing.T) { + tbl := newTestInodeTable(t) + mgr, err := NewManager(DefaultConfig(), tbl, nil) + if err != nil { + t.Fatal(err) + } + + if _, err := mgr.GetInode(inode.InodeFirstUsable); err != inode.ErrInodeNotInUse { + t.Fatalf("GetInode on free slot: err=%v, want ErrInodeNotInUse", err) + } + + mgr.CacheInode(claimWithSize(t, tbl, inode.InodeFirstUsable, 5)) + + got, err := mgr.GetInode(inode.InodeFirstUsable) + if err != nil { + t.Fatalf("GetInode after CacheInode: %v", err) + } + if got.Size != 5 { + t.Errorf("GetInode returned Size %d, want 5", got.Size) + } +} diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index b138edb5..eeeb3447 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -1840,6 +1840,8 @@ func (f *File) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, s return nil, 0, syscall.EISDIR } log.Printf("[DEBUG] File.Open: success, Size=%d", ino.Size) + // Close-to-open consistency: reads through this handle start from the on-disk inode. + f.fs.cacheInode(ino) // Return nil FileHandle - the File node itself implements Read/Write/etc. // The fuse.FOPEN_KEEP_CACHE flag tells the kernel to keep cached data valid // (we don't use it in cluster mode since other nodes may modify the file). diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 8c1c1c8a..d67d55d6 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -260,14 +260,7 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { return nil, ErrInodeInUse } - // Initialize the inode - newInode := &Inode{ - Magic: InodeMagic, - InodeNum: num, - LinkCount: 1, - LastModifierUUID: t.nodeUUID, - } - + newInode := t.newInodeInSlot(num, existing) if err := t.writeInodeUnlocked(newInode); err != nil { return nil, fmt.Errorf("failed to write new inode: %w", err) } @@ -275,6 +268,19 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { return newInode, nil } +// newInodeInSlot initializes an inode for a free slot. Its generations are higher +// than any previous occupant's, so caches on other nodes can detect the reuse. +func (t *Table) newInodeInSlot(num uint32, free *Inode) *Inode { + return &Inode{ + Magic: InodeMagic, + InodeNum: num, + LinkCount: 1, + DataGeneration: free.DataGeneration + 1, + MetaGeneration: free.MetaGeneration + 1, + LastModifierUUID: t.nodeUUID, + } +} + // Alloc allocates a new inode. // Returns the allocated inode with InodeNum set, ready for initialization. // @@ -293,13 +299,7 @@ func (t *Table) Alloc(ctx context.Context) (*Inode, error) { // Check if inode is free (Magic == 0) if inode.Magic == 0 { - // Found a free slot - initialize it - newInode := &Inode{ - Magic: InodeMagic, - InodeNum: num, - LinkCount: 1, - LastModifierUUID: t.nodeUUID, - } + newInode := t.newInodeInSlot(num, inode) // Acquire distributed lock if available if t.lockMgr != nil { @@ -352,9 +352,11 @@ func (t *Table) Free(ctx context.Context, inodeNum uint32) error { return ErrInodeNotInUse } - // Clear the inode (set Magic to 0) + // Keep the generations so the slot's next occupant gets higher ones. emptyInode := &Inode{ - InodeNum: inodeNum, + InodeNum: inodeNum, + DataGeneration: inode.DataGeneration, + MetaGeneration: inode.MetaGeneration, } return t.writeInodeUnlocked(emptyInode) @@ -567,11 +569,12 @@ func deserializeInode(buf []byte) (*Inode, error) { return nil, ErrInvalidInode } - // Check for empty inode (all zeros = free) magic := binary.LittleEndian.Uint32(buf[0:4]) if magic == 0 { - // Return empty inode with just the position - return &Inode{}, nil + return &Inode{ + DataGeneration: binary.LittleEndian.Uint64(buf[72:80]), + MetaGeneration: binary.LittleEndian.Uint64(buf[80:88]), + }, nil } // Verify checksum diff --git a/internal/journal/cross_node_recovery.go b/internal/journal/cross_node_recovery.go index 5c5514aa..c60a1b1f 100644 --- a/internal/journal/cross_node_recovery.go +++ b/internal/journal/cross_node_recovery.go @@ -183,6 +183,10 @@ func applyInodeCreate(ctx context.Context, rctx *RecoveryContext, payload []byte GID: p.GID, LinkCount: 1, } + if err == nil { + ino.DataGeneration = existing.DataGeneration + 1 + ino.MetaGeneration = existing.MetaGeneration + 1 + } now := time.Now().UnixNano() ino.Atime = now diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index f38d40fb..6c4376da 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2171,6 +2171,48 @@ test_broadcast_bidirectional() { return 0 } +# Test: Read after cross-node inode reuse +# Node 0 deletes a file node 1 has read and creates a new file with the same inode number. +# Node 1 must read the new file's full content, not the old size or old data block. +test_inode_reuse_stale_read() { + log "Testing read after cross-node inode reuse..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]}" + local dir=/mnt/goclusterfs + local old="reuse_old_$(date +%s).bin" + + exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' o > $dir/$old" + local old_ino + old_ino=$(exec_on_node "$writer" stat -c %i "$dir/$old") + exec_on_node "$reader" sh -c "cat $dir/$old > /dev/null" + + # New files take the lowest free inode, so create files until one gets the old number. + local new + new=$(exec_on_node "$writer" sh -c "rm $dir/$old; for i in \$(seq 1 200); do f=reuse_new_\$i.bin; touch $dir/\$f; if [ \$(stat -c %i $dir/\$f) = $old_ino ]; then echo \$f; break; fi; done") + if [[ -z "$new" ]]; then + log "ERROR: No new file reused inode $old_ino" + exec_on_node "$writer" sh -c "rm -f $dir/reuse_new_*.bin" + return 1 + fi + + exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' p > $dir/reuse_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$new" + # The healthcheck's ls can leave a pre-write size in the reader's 1s kernel attribute cache. + sleep 1.5 + local want got + want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$new") + got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$new") + exec_on_node "$writer" sh -c "rm -f $dir/reuse_new_*.bin $dir/reuse_pad.bin" + + if [[ "$got" != "$want" ]]; then + log "ERROR: $reader read wrong data from $new (inode $old_ino): md5 '$got', want '$want'" + return 1 + fi + + log "Inode reuse stale read test passed" + return 0 +} + # Test 4: Rolling restart test_rolling_restart() { log "Testing rolling restart..." @@ -2576,6 +2618,7 @@ run_test "Broadcast Invalidation" test_broadcast_invalidation run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidation run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional +run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read run_test "Rolling Restart" test_rolling_restart run_test "Stress Random Failures" test_stress_random_failures diff --git a/tests/integration/helpers.go b/tests/integration/helpers.go index 57df1d77..409c10a9 100644 --- a/tests/integration/helpers.go +++ b/tests/integration/helpers.go @@ -17,6 +17,7 @@ import ( type MountedFS struct { ImagePath string MountPoint string + ExtraArgs []string // Appended to the mount command by Mount MountCmd *exec.Cmd BinaryPath string t *testing.T @@ -190,6 +191,7 @@ func (m *MountedFS) Mount(debug bool) error { if debug { args = append(args, "-debug") } + args = append(args, m.ExtraArgs...) // Create a fresh context for this mount operation m.ctx, m.cancel = context.WithCancel(context.Background()) diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index 68e2e583..f421b0c4 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math/rand" "sync" + "syscall" "testing" "time" ) @@ -1054,6 +1055,85 @@ func TestMultiNodeReversedWriteOrder(t *testing.T) { t.Logf("Reversed write order test completed - file size: %d bytes", len(fullContent)) } +// TestMultiNodeInodeReuseStaleRead checks that a node which read a file reads the +// right data after another node deletes it and a new file reuses its inode number. +func TestMultiNodeInodeReuseStaleRead(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + // The reader must rely on its own cache validation, like VMs on separate networks. + cluster.Node(1).ExtraArgs = []string{"-broadcast=false"} + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + t.Run("SmallFile", func(t *testing.T) { + readAfterInodeReuse(t, writer, reader, "small", + []byte("v5\n"), []byte("from vm 1\n"), nil) + }) + + // The padding file pushes the new data away from the old block, so a stale extent mapping reads wrong data. + t.Run("BlockFile", func(t *testing.T) { + readAfterInodeReuse(t, writer, reader, "block", + bytes.Repeat([]byte("o"), 200<<10), + bytes.Repeat([]byte("n"), 300<<10), + bytes.Repeat([]byte("p"), 200<<10)) + }) +} + +func readAfterInodeReuse(t *testing.T, writer, reader *MountedFS, prefix string, oldData, newData, padData []byte) { + t.Helper() + oldName, newName, padName := prefix+"_old", prefix+"_new", prefix+"_pad" + + if err := writer.WriteFile(oldName, oldData); err != nil { + t.Fatalf("Writer: create %s: %v", oldName, err) + } + oldIno := inodeNumber(t, writer, oldName) + + got, err := reader.ReadFile(oldName) + if err != nil || !bytes.Equal(got, oldData) { + t.Fatalf("Reader: read %s: %d bytes, err=%v, want %d bytes", oldName, len(got), err, len(oldData)) + } + + if err := writer.Remove(oldName); err != nil { + t.Fatalf("Writer: remove %s: %v", oldName, err) + } + if err := writer.WriteFile(newName, nil); err != nil { + t.Fatalf("Writer: create %s: %v", newName, err) + } + if ino := inodeNumber(t, writer, newName); ino != oldIno { + t.Fatalf("%s got inode %d, want reused inode %d", newName, ino, oldIno) + } + if padData != nil { + if err := writer.WriteFile(padName, padData); err != nil { + t.Fatalf("Writer: create %s: %v", padName, err) + } + } + if err := writer.WriteFile(newName, newData); err != nil { + t.Fatalf("Writer: write %s: %v", newName, err) + } + + got, err = reader.ReadFile(newName) + if err != nil { + t.Fatalf("Reader: read %s: %v", newName, err) + } + if !bytes.Equal(got, newData) { + t.Errorf("Reader: %s has %d bytes starting %q, want %d bytes starting %q", + newName, len(got), got[:min(len(got), 16)], len(newData), newData[:min(len(newData), 16)]) + } +} + +func inodeNumber(t *testing.T, m *MountedFS, name string) uint64 { + t.Helper() + fi, err := m.Stat(name) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + return fi.Sys().(*syscall.Stat_t).Ino +} + // TestNetworkPartition simulates network partition handling by testing node isolation // and reconnection. Since Go integration tests run multiple FUSE mount processes on // the same host (sharing a backing image), true network partition cannot be simulated. diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index d429cf0c..2181ae0b 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -331,6 +331,15 @@ scenarios: docker: run-tests.sh::test_reversed_write_order vm: run-vm-tests.sh::test_reversed_write_order + - id: multinode-inode-reuse-stale-read + name: Read after cross-node inode reuse + description: One node deletes a file another node has read and creates a file that reuses its inode number; the other node reads the new file's full content + priority: critical + environments: + go: multinode_test.go::TestMultiNodeInodeReuseStaleRead + docker: run-tests.sh::test_inode_reuse_stale_read + vm: run-vm-tests.sh::test_inode_reuse_stale_read + # =========================================================================== # NETWORK AND RESILIENCE TESTS # =========================================================================== @@ -848,6 +857,20 @@ scenarios: environments: go: internal/cache/broadcast/broadcaster_test.go::TestBroadcasterHandleInvalidateAll + - id: inode-cache-slot-reuse + name: Inode cache detects inode slot reuse + description: After an inode slot is freed and claimed again, cache validation returns the new inode + priority: go_only + environments: + go: internal/cache/inode_cache_test.go::TestGetInodeDetectsSlotReuse + + - id: inode-cache-clears-negative + name: Caching an inode clears its negative entry + description: CacheInode replaces an earlier not-found result for the same inode number + priority: go_only + environments: + go: internal/cache/inode_cache_test.go::TestCacheInodeClearsNegativeEntry + - id: broadcast-ignore-own name: Broadcast ignores own messages description: Broadcaster ignores messages from itself diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 3fd08de3..82bdfb76 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -1566,6 +1566,53 @@ test_broadcast_bidirectional() { return 0 } +# Test: Read after cross-node inode reuse +# VM 1 deletes a file VM 2 has read and creates a new file with the same inode number. +# VM 2 must read the new file's full content, not the old size or old data block. +test_inode_reuse_stale_read() { + log_info "Testing read after cross-node inode reuse..." + + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping inode reuse test (requires 2+ VMs)" + return 0 + fi + + ensure_cluster_healthy 2 + + local writer=1 + local reader=2 + local dir=/mnt/goclusterfs + local old="reuse_old_$(date +%s).bin" + + ssh_run "$writer" "head -c 204800 /dev/zero | tr '\\0' o > $dir/$old" + local old_ino + old_ino=$(ssh_run "$writer" "stat -c %i $dir/$old") + ssh_run "$reader" "cat $dir/$old > /dev/null" + + # New files take the lowest free inode, so create files until one gets the old number. + local new + new=$(ssh_run "$writer" "rm $dir/$old; for i in \$(seq 1 200); do f=reuse_new_\$i.bin; touch $dir/\$f; if [ \$(stat -c %i $dir/\$f) = $old_ino ]; then echo \$f; break; fi; done") + if [[ -z "$new" ]]; then + log_error "No new file reused inode $old_ino" + ssh_run "$writer" "rm -f $dir/reuse_new_*.bin" + return 1 + fi + + ssh_run "$writer" "head -c 204800 /dev/zero | tr '\\0' p > $dir/reuse_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$new" + local want got + want=$(ssh_run "$writer" "md5sum < $dir/$new") + got=$(ssh_run "$reader" "md5sum < $dir/$new") + ssh_run "$writer" "rm -f $dir/reuse_new_*.bin $dir/reuse_pad.bin" + + if [[ "$got" != "$want" ]]; then + log_error "VM $reader read wrong data from $new (inode $old_ino): md5 '$got', want '$want'" + return 1 + fi + + log_info "Inode reuse stale read test passed" + return 0 +} + # Test 4: Rolling restart test_rolling_restart() { log_info "Testing rolling restart..." @@ -2687,6 +2734,7 @@ main() { run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidation run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional + run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read run_test "Rolling Restart" test_rolling_restart run_test "Stress Random Failures" test_stress_random_failures