From ff6ae8f357312f81e23cfb52b3bec9d26fa064c6 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:12:41 -0500 Subject: [PATCH 1/6] fix(cache): keep extent mappings and inodes coherent across nodes Cached extent mappings were validated only against DataGeneration, which ordinary writes never change, so a node kept using a mapping after another node truncated, rewrote, hole-filled or CoW-copied the file. Every mapping change is committed by an inode write, which bumps MetaGeneration, so mappings now carry both generations and miss for any later inode version. The extent invalidation hooks, which were no-ops, now drop the cached inode so the next lookup reads the current version. CacheInode no longer replaces a newer cached copy, so a disk read in File.Open that finishes after a local write cannot install the older inode. The kernel attribute and entry timeouts are now 0. go-fuse replaced the AttrValid/EntryValid of 0 that GoClusterFS returns with the 1s mount option, so a node could serve a stale size for up to 1s after another node wrote the file. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 4 +- docs/API-CONFIG.md | 4 +- internal/cache/cache.go | 9 ++++ internal/cache/extent_cache.go | 91 ++++++++++++---------------------- internal/cache/inode_cache.go | 9 ++-- internal/cache/lru.go | 17 +++++++ internal/cache/sharded_lru.go | 5 ++ 7 files changed, 73 insertions(+), 66 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 7d13bd36..c70d21d4 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -323,8 +323,8 @@ func main() { // Configure go-fuse mount options // AttrTimeout and EntryTimeout control kernel-side caching - attrTimeout := 1 * time.Second - entryTimeout := 1 * time.Second + attrTimeout := time.Duration(0) + entryTimeout := time.Duration(0) mountOpts := &fs.Options{ MountOptions: fuse.MountOptions{ diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 741784a8..12fddb64 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -43,6 +43,8 @@ These flags are passed when starting `goclusterfs`: | `-cache-strict` | bool | `false` | Enable strict cache mode (always validate against disk) | | `-cache-ttl` | duration | `5s` | Cache entry TTL before revalidation required | +The kernel does not cache file attributes or directory entries: the mount sets the FUSE attribute and entry timeouts to 0, so every `stat` and path lookup reaches GoClusterFS. Opening a file reads its inode from disk, so a node sees another node's completed writes when it next opens the file. Reads through a file that is already open use the inode cache and see other nodes' changes within `-cache-ttl`, or sooner when broadcast is enabled. + ### Journal Flags | Flag | Type | Default | Description | @@ -143,7 +145,7 @@ Time after which a cached inode must be revalidated against disk. | Memory | ~3.2 MB (for defaults) | | Recommended | 50,000 - 500,000 | -The maximum number of extent mappings to cache. Extent cache maps (inode, logical block) → physical block. +The maximum number of extent mappings to cache. Extent cache maps (inode, logical block) → physical block. A cached mapping is only used with the inode version it was looked up with, so any inode write by any node makes it stale. **Sizing guidance**: - Large files benefit more from extent caching diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 2df642d5..3b0ecc4f 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -36,6 +36,15 @@ func (c *CachedInode) IsStale(diskDataGen, diskMetaGen uint64) bool { return c.DataGeneration != diskDataGen || c.MetaGeneration != diskMetaGen } +// newerThan reports whether c holds a later version of the inode than in. +// DataGeneration only changes when the inode number is reused, so it orders first. +func (c *CachedInode) newerThan(in *inode.Inode) bool { + if c.DataGeneration != in.DataGeneration { + return c.DataGeneration > in.DataGeneration + } + return c.MetaGeneration > in.MetaGeneration +} + // NeedsValidation returns true if the cache entry needs revalidation against disk. func (c *CachedInode) NeedsValidation(ttl time.Duration, strictMode bool) bool { if strictMode { diff --git a/internal/cache/extent_cache.go b/internal/cache/extent_cache.go index 0d39f776..c2d281c3 100644 --- a/internal/cache/extent_cache.go +++ b/internal/cache/extent_cache.go @@ -20,27 +20,23 @@ type CachedExtent struct { // A value of 0 means "hole" (sparse file - unallocated region). PhysicalBlock uint64 - // InodeDataGen is the inode's DataGeneration at cache time. - // When the inode's DataGeneration changes, all extent cache - // entries for that inode become stale. + // InodeDataGen and InodeMetaGen are the inode's generations at cache time. + // Every extent mapping change is committed by an inode write, which bumps + // MetaGeneration, so a mapping is only valid for the inode version it was + // looked up with. InodeDataGen uint64 + InodeMetaGen uint64 // CachedAt is when this entry was cached. CachedAt time.Time } -// IsStale returns true if the cached extent is stale relative to the inode. -func (c *CachedExtent) IsStale(currentDataGen uint64) bool { - return c.InodeDataGen != currentDataGen +// IsStale returns true if the cached extent was looked up with a different inode version. +func (c *CachedExtent) IsStale(dataGen, metaGen uint64) bool { + return c.InodeDataGen != dataGen || c.InodeMetaGen != metaGen } -// GetExtent retrieves a cached extent mapping. -// -// The retrieval flow is: -// 1. Check extent cache for (inodeNum, logicalBlock) -// 2. If found, validate against inode's DataGeneration -// 3. If valid, return cached physical block -// 4. If stale or not found, return cache miss +// GetExtent retrieves a cached extent mapping for this version of ino. // // This function does NOT perform disk I/O - caller must fall back to // extent.Manager.LookupPhysical() on cache miss. @@ -48,9 +44,9 @@ func (c *CachedExtent) IsStale(currentDataGen uint64) bool { // Returns: // - physicalBlock: The physical block number (0 = hole) // - found: true if cache hit and valid -func (m *Manager) GetExtent(inodeNum uint32, logicalBlock uint64, inodeDataGen uint64) (physicalBlock uint64, found bool) { +func (m *Manager) GetExtent(ino *inode.Inode, logicalBlock uint64) (physicalBlock uint64, found bool) { key := ExtentCacheKey{ - InodeNum: inodeNum, + InodeNum: ino.InodeNum, LogicalBlock: logicalBlock, } @@ -59,9 +55,7 @@ func (m *Manager) GetExtent(inodeNum uint32, logicalBlock uint64, inodeDataGen u return 0, false } - // Validate against inode's current DataGeneration - if cached.IsStale(inodeDataGen) { - // Stale entry - remove it + if cached.IsStale(ino.DataGeneration, ino.MetaGeneration) { m.extents.Delete(key) return 0, false } @@ -69,69 +63,50 @@ func (m *Manager) GetExtent(inodeNum uint32, logicalBlock uint64, inodeDataGen u return cached.PhysicalBlock, true } -// CacheExtent stores an extent mapping in the cache. +// CacheExtent stores an extent mapping looked up with this version of ino. // // Call this after a successful extent.Manager.LookupPhysical() to cache -// the result for future reads. -// -// Parameters: -// - inodeNum: The inode number -// - logicalBlock: The logical file block being looked up -// - physicalBlock: The physical disk block (0 for hole) -// - inodeDataGen: The inode's current DataGeneration -func (m *Manager) CacheExtent(inodeNum uint32, logicalBlock uint64, physicalBlock uint64, inodeDataGen uint64) { +// the result for future reads. physicalBlock is 0 for a hole. +func (m *Manager) CacheExtent(ino *inode.Inode, logicalBlock uint64, physicalBlock uint64) { if m.config.ExtentCacheSize == 0 { return // Extent cache disabled } key := ExtentCacheKey{ - InodeNum: inodeNum, + InodeNum: ino.InodeNum, LogicalBlock: logicalBlock, } cached := &CachedExtent{ PhysicalBlock: physicalBlock, - InodeDataGen: inodeDataGen, + InodeDataGen: ino.DataGeneration, + InodeMetaGen: ino.MetaGeneration, CachedAt: time.Now(), } m.extents.Put(key, cached) } -// InvalidateExtents removes all cached extents for an inode. +// InvalidateExtents makes cached extents for an inode stale. // -// Call this when: -// - File is truncated (all extents may change) -// - File data is modified in a way that changes extent mapping -// - Inode's DataGeneration has changed -// -// Note: Individual extent invalidation is handled lazily via generation -// check in GetExtent. This function is for bulk invalidation. +// Call this when another node changed the inode's extent mapping (truncate, +// hole fill, CoW). It drops the cached inode, so the next lookup reads the +// current inode version from disk and every mapping cached for an older +// version misses. func (m *Manager) InvalidateExtents(inodeNum uint32) { - // Unfortunately, our LRU doesn't support prefix/range deletion. - // We rely on lazy invalidation via DataGeneration checks. - // For bulk invalidation, we'd need to iterate all entries. - // - // For now, extents are invalidated lazily when GetExtent detects - // a DataGeneration mismatch. This is acceptable because: - // 1. Stale entries don't return incorrect data (generation check) - // 2. They get evicted naturally by LRU or replaced on next lookup - // 3. InvalidateInodes() already handles the common case + m.inodes.Delete(inodeNum) } -// InvalidateExtentsForInodes removes cached extents when inode generation changes. -// This is called from OnLockAcquire for extent-level locks. +// InvalidateExtentsForInodes makes cached extents for several inodes stale. func (m *Manager) InvalidateExtentsForInodes(inodeNums ...uint32) { - // Lazy invalidation via generation check handles this. - // See InvalidateExtents() comment. + for _, inodeNum := range inodeNums { + m.InvalidateExtents(inodeNum) + } } -// InvalidateExtentsForInode removes cached extents for a single inode. +// InvalidateExtentsForInode makes cached extents for a single inode stale. // This satisfies the broadcast.InvalidationHandler interface. func (m *Manager) InvalidateExtentsForInode(inodeNum uint32) { - // Lazy invalidation via DataGeneration check handles this. - // When the next GetExtent() call is made, the generation mismatch - // will cause the stale entry to be evicted. m.InvalidateExtents(inodeNum) } @@ -142,26 +117,24 @@ func (m *Manager) InvalidateExtentsForInode(inodeNum uint32) { // // Parameters: // - extMgr: The extent manager for disk fallback -// - ino: The inode (must have current DataGeneration) +// - ino: A committed inode version (no uncommitted extent edits) // - logicalBlock: The logical file block to look up // // Returns: // - physicalBlock: The physical disk block (0 = hole) // - err: Any error from disk lookup func (m *Manager) LookupPhysicalCached(extMgr *extent.Manager, ino *inode.Inode, logicalBlock uint64) (uint64, error) { - // Try cache first - if phys, found := m.GetExtent(ino.InodeNum, logicalBlock, ino.DataGeneration); found { + if phys, found := m.GetExtent(ino, logicalBlock); found { return phys, nil } - // Cache miss - look up from disk phys, err := extMgr.LookupPhysical(ino, logicalBlock) if err != nil { return 0, err } // Cache the result (including holes which have phys=0) - m.CacheExtent(ino.InodeNum, logicalBlock, phys, ino.DataGeneration) + m.CacheExtent(ino, logicalBlock, phys) return phys, nil } diff --git a/internal/cache/inode_cache.go b/internal/cache/inode_cache.go index 7d075d5c..276aafc6 100644 --- a/internal/cache/inode_cache.go +++ b/internal/cache/inode_cache.go @@ -100,13 +100,14 @@ func (m *Manager) InvalidateAll() { } // CacheInode explicitly caches an inode. -// Use this after reading an inode from disk in a write path, -// so subsequent reads can benefit from the cached data. +// Use this after reading or writing an inode, so subsequent reads can +// benefit from the cached data. A newer cached copy is kept. func (m *Manager) CacheInode(in *inode.Inode) { m.cacheInode(in) } -// cacheInode adds an inode to the cache (internal helper). +// cacheInode adds an inode to the cache unless a newer copy is already cached. +// A disk read can finish after a local write cached a later version. func (m *Manager) cacheInode(in *inode.Inode) { now := time.Now() cached := &CachedInode{ @@ -116,7 +117,7 @@ func (m *Manager) cacheInode(in *inode.Inode) { CachedAt: now, LastValidated: now, } - m.inodes.Put(in.InodeNum, cached) + m.inodes.PutIf(in.InodeNum, cached, func(old *CachedInode) bool { return !old.newerThan(in) }) m.negatives.Delete(in.InodeNum) } diff --git a/internal/cache/lru.go b/internal/cache/lru.go index e0f65c24..02928562 100644 --- a/internal/cache/lru.go +++ b/internal/cache/lru.go @@ -85,6 +85,23 @@ func (l *LRU[K, V]) Put(key K, value V) (evictedKey K, evictedValue V, evicted b l.mu.Lock() defer l.mu.Unlock() + return l.putLocked(key, value) +} + +// PutIf stores value unless key is already present and replace(old) returns false. +// The check and the store are atomic. Returns whether value was stored. +func (l *LRU[K, V]) PutIf(key K, value V, replace func(old V) bool) bool { + l.mu.Lock() + defer l.mu.Unlock() + + if elem, exists := l.items[key]; exists && !replace(elem.Value.(*lruEntry[K, V]).value) { + return false + } + l.putLocked(key, value) + return true +} + +func (l *LRU[K, V]) putLocked(key K, value V) (evictedKey K, evictedValue V, evicted bool) { if l.capacity == 0 { return } diff --git a/internal/cache/sharded_lru.go b/internal/cache/sharded_lru.go index e13303d0..d7f47fb6 100644 --- a/internal/cache/sharded_lru.go +++ b/internal/cache/sharded_lru.go @@ -67,6 +67,11 @@ func (s *ShardedLRU[K, V]) Put(key K, value V) (evictedKey K, evictedValue V, ev return s.shards[s.shard(key)].Put(key, value) } +// PutIf stores value unless key is already present and replace(old) returns false. +func (s *ShardedLRU[K, V]) PutIf(key K, value V, replace func(old V) bool) bool { + return s.shards[s.shard(key)].PutIf(key, value, replace) +} + // Delete removes an entry from the cache. func (s *ShardedLRU[K, V]) Delete(key K) (V, bool) { return s.shards[s.shard(key)].Delete(key) From fcba4f30d7e9bd782af7cbb82631942f180b9b06 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:12:41 -0500 Subject: [PATCH 2/6] test: cover cross-node extent cache coherency Co-Authored-By: Claude Opus 5.5 --- internal/cache/cache_test.go | 29 ++++++------ internal/cache/extent_cache_test.go | 38 ++++++++++++++++ internal/cache/inode_cache_test.go | 33 ++++++++++++++ tests/docker/scripts/run-tests.sh | 41 ++++++++++++++++- tests/integration/multinode_test.go | 70 +++++++++++++++++++++++++++++ tests/test-manifest.yaml | 30 +++++++++++++ tests/vm/scripts/run-vm-tests.sh | 46 +++++++++++++++++++ 7 files changed, 271 insertions(+), 16 deletions(-) create mode 100644 internal/cache/extent_cache_test.go diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 52d27f06..2591a4ed 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -381,16 +381,17 @@ func TestCachedExtent(t *testing.T) { cached := &CachedExtent{ PhysicalBlock: 100, InodeDataGen: 5, + InodeMetaGen: 7, } - // Same generation - not stale - if cached.IsStale(5) { - t.Error("IsStale() should return false when generation matches") + if cached.IsStale(5, 7) { + t.Error("IsStale() should return false when generations match") } - - // Different generation - stale - if !cached.IsStale(6) { - t.Error("IsStale() should return true when generation differs") + if !cached.IsStale(6, 7) { + t.Error("IsStale() should return true when DataGeneration differs") + } + if !cached.IsStale(5, 8) { + t.Error("IsStale() should return true when MetaGeneration differs") } }) } @@ -491,14 +492,14 @@ func TestExtentCache(t *testing.T) { t.Fatal("Get() should find cached extent") } - // Same generation - not stale - if got.IsStale(10) { - t.Error("Extent should not be stale with matching generation") + if got.IsStale(10, 0) { + t.Error("Extent should not be stale with matching generations") } - - // Different generation - stale - if !got.IsStale(11) { - t.Error("Extent should be stale with different generation") + if !got.IsStale(11, 0) { + t.Error("Extent should be stale with different DataGeneration") + } + if !got.IsStale(10, 1) { + t.Error("Extent should be stale with different MetaGeneration") } }) diff --git a/internal/cache/extent_cache_test.go b/internal/cache/extent_cache_test.go new file mode 100644 index 00000000..0d5ccd7d --- /dev/null +++ b/internal/cache/extent_cache_test.go @@ -0,0 +1,38 @@ +package cache + +import ( + "testing" + + "github.com/supporttools/GoClusterFS/internal/inode" +) + +func TestGetExtentMissesAfterInodeWrite(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + v1 := &inode.Inode{InodeNum: 3, DataGeneration: 1, MetaGeneration: 4} + mgr.CacheExtent(v1, 0, 100) + if phys, ok := mgr.GetExtent(v1, 0); !ok || phys != 100 { + t.Fatalf("GetExtent with the caching version = (%d, %v), want (100, true)", phys, ok) + } + + v2 := &inode.Inode{InodeNum: 3, DataGeneration: 1, MetaGeneration: 5} + if phys, ok := mgr.GetExtent(v2, 0); ok { + t.Errorf("GetExtent after an inode write hit with block %d, want a miss", phys) + } +} + +func TestInvalidateExtentsForInodeDropsCachedInode(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + mgr.CacheInode(&inode.Inode{InodeNum: 3, MetaGeneration: 1}) + mgr.InvalidateExtentsForInode(3) + if mgr.IsCached(3) { + t.Error("inode 3 still cached after InvalidateExtentsForInode") + } +} diff --git a/internal/cache/inode_cache_test.go b/internal/cache/inode_cache_test.go index 7839851f..dfb007d7 100644 --- a/internal/cache/inode_cache_test.go +++ b/internal/cache/inode_cache_test.go @@ -108,3 +108,36 @@ func TestCacheInodeClearsNegativeEntry(t *testing.T) { t.Errorf("GetInode returned Size %d, want 5", got.Size) } } + +func TestCacheInodeKeepsNewerCopy(t *testing.T) { + tbl := newTestInodeTable(t) + mgr, err := NewManager(DefaultConfig(), tbl, nil) + if err != nil { + t.Fatal(err) + } + + claimWithSize(t, tbl, inode.InodeFirstUsable, 3) + older, err := tbl.Read(inode.InodeFirstUsable) + if err != nil { + t.Fatal(err) + } + newer, err := tbl.Read(inode.InodeFirstUsable) + if err != nil { + t.Fatal(err) + } + newer.Size = 10 + if err := tbl.Write(context.Background(), newer); err != nil { + t.Fatal(err) + } + + mgr.CacheInode(newer) + mgr.CacheInode(older) + + got, err := mgr.GetInode(inode.InodeFirstUsable) + if err != nil { + t.Fatalf("GetInode: %v", err) + } + if got.Size != 10 { + t.Errorf("GetInode returned Size %d from the older copy, want 10", got.Size) + } +} diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 0f7406fc..9357b80c 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2197,8 +2197,6 @@ test_inode_reuse_stale_read() { fi exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' p > $dir/reuse_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$new" - # The healthcheck's ls can leave a pre-write size in the reader's 1s kernel attribute cache. - sleep 1.5 local want got want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$new") got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$new") @@ -2213,6 +2211,44 @@ test_inode_reuse_stale_read() { return 0 } +# Test: Extent cache coherency across nodes +# Node 1 reads files, node 0 changes their block mappings (truncate and rewrite, +# hole fill), and node 1 must read the new data. +test_extent_cache_coherency() { + log "Testing extent cache coherency across nodes..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]}" + local dir=/mnt/goclusterfs + local trunc="extent_trunc_$(date +%s).bin" + local holes="extent_holes_$(date +%s).bin" + + exec_on_node "$writer" sh -c "head -c 307200 /dev/zero | tr '\\0' o > $dir/$trunc" + exec_on_node "$writer" sh -c "head -c 4096 /dev/zero | tr '\\0' a > $dir/$holes && head -c 4096 /dev/zero | tr '\\0' c | dd of=$dir/$holes bs=1M seek=2 conv=notrunc 2>/dev/null" + exec_on_node "$reader" sh -c "cat $dir/$trunc $dir/$holes > /dev/null" + + # The padding file takes the freed block, so the rewrite lands in another one. + exec_on_node "$writer" sh -c "truncate -s 0 $dir/$trunc && head -c 307200 /dev/zero | tr '\\0' p > $dir/extent_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$trunc" + exec_on_node "$writer" sh -c "head -c 4096 /dev/zero | tr '\\0' b | dd of=$dir/$holes bs=1M seek=1 conv=notrunc 2>/dev/null" + + local f want got failed=0 + for f in "$trunc" "$holes"; do + want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$f") + got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$f") + if [[ "$got" != "$want" ]]; then + log "ERROR: $reader read stale data from $f: md5 '$got', want '$want'" + failed=1 + fi + done + exec_on_node "$writer" sh -c "rm -f $dir/$trunc $dir/$holes $dir/extent_pad.bin" + + if [[ $failed -ne 0 ]]; then + return 1 + fi + log "Extent cache coherency test passed" + return 0 +} + # Test 4: Rolling restart test_rolling_restart() { log "Testing rolling restart..." @@ -2737,6 +2773,7 @@ run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidatio run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read +run_test "Extent Cache Coherency" test_extent_cache_coherency run_test "Rolling Restart" test_rolling_restart run_test "Stress Random Failures" test_stress_random_failures diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index f421b0c4..14ed8d9f 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -6,6 +6,7 @@ import ( "bytes" "fmt" "math/rand" + "os" "sync" "syscall" "testing" @@ -1134,6 +1135,75 @@ func inodeNumber(t *testing.T, m *MountedFS, name string) uint64 { return fi.Sys().(*syscall.Stat_t).Ino } +// TestMultiNodeExtentCacheCoherency checks that a node which cached a file's block +// mappings reads the new data after another node changes those mappings. +func TestMultiNodeExtentCacheCoherency(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + cluster.Node(1).ExtraArgs = []string{"-broadcast=false"} + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + t.Run("TruncateRewrite", func(t *testing.T) { + const name = "trunc.bin" + writeOrFail(t, writer, name, bytes.Repeat([]byte("o"), 300<<10), 0) + expectContent(t, reader, name, bytes.Repeat([]byte("o"), 300<<10)) + + if err := os.Truncate(writer.Path(name), 0); err != nil { + t.Fatalf("Writer: truncate %s: %v", name, err) + } + // The padding file takes the freed block, so the rewrite lands in another one. + writeOrFail(t, writer, "trunc_pad.bin", bytes.Repeat([]byte("p"), 300<<10), 0) + writeOrFail(t, writer, name, bytes.Repeat([]byte("n"), 300<<10), 0) + + expectContent(t, reader, name, bytes.Repeat([]byte("n"), 300<<10)) + }) + + t.Run("HoleFill", func(t *testing.T) { + const name = "holes.bin" + const blockSize = 1 << 20 + chunk := func(c byte) []byte { return bytes.Repeat([]byte{c}, 4096) } + writeOrFail(t, writer, name, chunk('a'), 0) + writeOrFail(t, writer, name, chunk('c'), 2*blockSize) + + want := make([]byte, 2*blockSize+4096) + copy(want, chunk('a')) + copy(want[2*blockSize:], chunk('c')) + expectContent(t, reader, name, want) + + writeOrFail(t, writer, name, chunk('b'), blockSize) + copy(want[blockSize:], chunk('b')) + expectContent(t, reader, name, want) + }) +} + +func writeOrFail(t *testing.T, m *MountedFS, name string, data []byte, offset int64) { + t.Helper() + if err := m.WriteFileAt(name, data, offset); err != nil { + t.Fatalf("write %s at %d: %v", name, offset, err) + } +} + +func expectContent(t *testing.T, m *MountedFS, name string, want []byte) { + t.Helper() + got, err := m.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + if bytes.Equal(got, want) { + return + } + i := 0 + for i < len(got) && i < len(want) && got[i] == want[i] { + i++ + } + t.Errorf("%s: %d bytes, want %d; first difference at offset %d", name, len(got), len(want), i) +} + // TestNetworkPartition simulates network partition handling by testing node isolation // and reconnection. Since Go integration tests run multiple FUSE mount processes on // the same host (sharing a backing image), true network partition cannot be simulated. diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index a6ade5ac..ac0e9789 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -340,6 +340,15 @@ scenarios: docker: run-tests.sh::test_inode_reuse_stale_read vm: run-vm-tests.sh::test_inode_reuse_stale_read + - id: multinode-extent-cache-coherency + name: Extent cache coherency across nodes + description: One node changes the block mappings of files another node has read (truncate and rewrite, hole fill); the other node reads the new data + priority: critical + environments: + go: multinode_test.go::TestMultiNodeExtentCacheCoherency + docker: run-tests.sh::test_extent_cache_coherency + vm: run-vm-tests.sh::test_extent_cache_coherency + # =========================================================================== # NETWORK AND RESILIENCE TESTS # =========================================================================== @@ -965,6 +974,27 @@ scenarios: environments: go: internal/cache/inode_cache_test.go::TestCacheInodeClearsNegativeEntry + - id: inode-cache-keeps-newer + name: Inode cache keeps the newer copy + description: Caching an older copy of an inode does not replace a newer cached copy + priority: go_only + environments: + go: internal/cache/inode_cache_test.go::TestCacheInodeKeepsNewerCopy + + - id: extent-cache-inode-version + name: Extent cache misses after an inode write + description: A cached block mapping is not used with a later version of its inode + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestGetExtentMissesAfterInodeWrite + + - id: extent-cache-invalidate + name: Extent invalidation drops the cached inode + description: InvalidateExtentsForInode forces the next lookup to read the current inode version + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestInvalidateExtentsForInodeDropsCachedInode + - id: broadcast-ignore-own name: Broadcast ignores own messages description: Broadcaster ignores messages from itself diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index aafbc100..097349ca 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -1613,6 +1613,51 @@ test_inode_reuse_stale_read() { return 0 } +# Test: Extent cache coherency across VMs +# VM 2 reads files, VM 1 changes their block mappings (truncate and rewrite, +# hole fill), and VM 2 must read the new data. +test_extent_cache_coherency() { + log_info "Testing extent cache coherency across VMs..." + + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping extent cache coherency test (requires 2+ VMs)" + return 0 + fi + + ensure_cluster_healthy 2 + + local writer=1 + local reader=2 + local dir=/mnt/goclusterfs + local trunc="extent_trunc_$(date +%s).bin" + local holes="extent_holes_$(date +%s).bin" + + ssh_run "$writer" "head -c 307200 /dev/zero | tr '\\0' o > $dir/$trunc" + ssh_run "$writer" "head -c 4096 /dev/zero | tr '\\0' a > $dir/$holes && head -c 4096 /dev/zero | tr '\\0' c | dd of=$dir/$holes bs=1M seek=2 conv=notrunc 2>/dev/null" + ssh_run "$reader" "cat $dir/$trunc $dir/$holes > /dev/null" + + # The padding file takes the freed block, so the rewrite lands in another one. + ssh_run "$writer" "truncate -s 0 $dir/$trunc && head -c 307200 /dev/zero | tr '\\0' p > $dir/extent_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$trunc" + ssh_run "$writer" "head -c 4096 /dev/zero | tr '\\0' b | dd of=$dir/$holes bs=1M seek=1 conv=notrunc 2>/dev/null" + + local f want got failed=0 + for f in "$trunc" "$holes"; do + want=$(ssh_run "$writer" "md5sum < $dir/$f") + got=$(ssh_run "$reader" "md5sum < $dir/$f") + if [[ "$got" != "$want" ]]; then + log_error "VM $reader read stale data from $f: md5 '$got', want '$want'" + failed=1 + fi + done + ssh_run "$writer" "rm -f $dir/$trunc $dir/$holes $dir/extent_pad.bin" + + if [[ $failed -ne 0 ]]; then + return 1 + fi + log_info "Extent cache coherency test passed" + return 0 +} + # Test 4: Rolling restart test_rolling_restart() { log_info "Testing rolling restart..." @@ -2856,6 +2901,7 @@ main() { run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read + run_test "Extent Cache Coherency" test_extent_cache_coherency run_test "Rolling Restart" test_rolling_restart run_test "Stress Random Failures" test_stress_random_failures From 4d9ee163a4f597b79f07055445380fd55d63d32c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 16:11:41 -0500 Subject: [PATCH 3/6] fix(cache): address review on cache coherency - Keep the kernel attribute and entry timeouts at 1s. At 0, every stat and path step runs a directory lookup with a cluster lock; a stat loop on one file went from 0.01s to 13.9s. Document the 1s window and what an open file can read within -cache-ttl. - Broadcast the invalidation before caching the new inode in the write paths, so a concurrent read cannot reinsert an older inode after the local invalidation. - Count a stale extent lookup as a miss instead of a hit. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 4 ++-- docs/API-CONFIG.md | 2 +- internal/cache/cache.go | 3 ++- internal/cache/extent_cache.go | 10 +++------- internal/cache/lru.go | 22 ++++++++++++++++++++++ internal/cache/sharded_lru.go | 5 +++++ internal/goclusterfs/fs.go | 22 +++++++++++----------- 7 files changed, 46 insertions(+), 22 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 514d80d9..a73fd483 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -321,8 +321,8 @@ func main() { // Configure go-fuse mount options // AttrTimeout and EntryTimeout control kernel-side caching - attrTimeout := time.Duration(0) - entryTimeout := time.Duration(0) + attrTimeout := 1 * time.Second + entryTimeout := 1 * time.Second mountOpts := &fs.Options{ MountOptions: fuse.MountOptions{ diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index ee251e55..2e5b1945 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -43,7 +43,7 @@ These flags are passed when starting `goclusterfs`: | `-cache-strict` | bool | `false` | Enable strict cache mode (always validate against disk) | | `-cache-ttl` | duration | `5s` | Cache entry TTL before revalidation required | -The kernel does not cache file attributes or directory entries: the mount sets the FUSE attribute and entry timeouts to 0, so every `stat` and path lookup reaches GoClusterFS. Opening a file reads its inode from disk, so a node sees another node's completed writes when it next opens the file. Reads through a file that is already open use the inode cache and see other nodes' changes within `-cache-ttl`, or sooner when broadcast is enabled. +The kernel caches file attributes and directory entries for 1s (the FUSE attribute and entry timeouts). For up to 1s after another node changes a file, `stat` and path lookups on this node can return the old size, and a read can stop at the old size even right after `open`. After that, opening a file reads its inode from disk, so the reader sees the other node's completed writes. Reads through a file that is already open use the inode cache and can use the old size and block mappings for up to `-cache-ttl`, including a block that another node has since freed and reused for other data. With broadcast enabled, writes and directory changes invalidate peers' caches sooner; truncate, fallocate and unlink do not. ### Journal Flags diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 3b0ecc4f..6f61164d 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -263,7 +263,8 @@ func (m *Manager) BroadcastEnabled() bool { } // BroadcastInvalidateInode invalidates an inode locally AND broadcasts -// the invalidation to peer nodes. Call this after modifying an inode. +// the invalidation to peer nodes. Call this after modifying an inode, and +// cache the new inode after this call, since it drops the local entry. // // Parameters: // - inodeNum: The inode number being invalidated diff --git a/internal/cache/extent_cache.go b/internal/cache/extent_cache.go index 82b6449a..b1754f74 100644 --- a/internal/cache/extent_cache.go +++ b/internal/cache/extent_cache.go @@ -49,16 +49,12 @@ func (m *Manager) GetExtent(ino *inode.Inode, logicalBlock uint64) (physicalBloc LogicalBlock: logicalBlock, } - cached, ok := m.extents.Get(key) + cached, ok := m.extents.GetIf(key, func(c *CachedExtent) bool { + return !c.IsStale(ino.DataGeneration, ino.MetaGeneration) + }) if !ok { return 0, false } - - if cached.IsStale(ino.DataGeneration, ino.MetaGeneration) { - m.extents.Delete(key) - return 0, false - } - return cached.PhysicalBlock, true } diff --git a/internal/cache/lru.go b/internal/cache/lru.go index 02928562..58bce5af 100644 --- a/internal/cache/lru.go +++ b/internal/cache/lru.go @@ -64,6 +64,28 @@ func (l *LRU[K, V]) Get(key K) (V, bool) { return zero, false } +// GetIf is Get for entries that can go stale: an entry for which valid +// returns false is removed and counted as a miss. +func (l *LRU[K, V]) GetIf(key K, valid func(V) bool) (V, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + if elem, exists := l.items[key]; exists { + entry := elem.Value.(*lruEntry[K, V]) + if valid(entry.value) { + l.order.MoveToFront(elem) + l.hits++ + return entry.value, true + } + l.order.Remove(elem) + delete(l.items, key) + } + + var zero V + l.misses++ + return zero, false +} + // Peek retrieves a value without affecting LRU ordering. // Use this for validation checks that shouldn't promote entries. func (l *LRU[K, V]) Peek(key K) (V, bool) { diff --git a/internal/cache/sharded_lru.go b/internal/cache/sharded_lru.go index d7f47fb6..53b96655 100644 --- a/internal/cache/sharded_lru.go +++ b/internal/cache/sharded_lru.go @@ -56,6 +56,11 @@ func (s *ShardedLRU[K, V]) Get(key K) (V, bool) { return s.shards[s.shard(key)].Get(key) } +// GetIf retrieves a value if valid accepts it; otherwise the entry is removed. +func (s *ShardedLRU[K, V]) GetIf(key K, valid func(V) bool) (V, bool) { + return s.shards[s.shard(key)].GetIf(key, valid) +} + // Peek retrieves a value without affecting LRU ordering. func (s *ShardedLRU[K, V]) Peek(key K) (V, bool) { return s.shards[s.shard(key)].Peek(key) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 140f756d..81274c49 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -724,12 +724,12 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, log.Printf("[DEBUG] Create %q: updating root inode failed: %v", name, err) // The file was created successfully, just log the error } else { - // Update cache with parent directory inode - f.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if f.cacheMgr != nil { f.cacheMgr.BroadcastInvalidateInode(uint32(inode.InodeRoot), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + f.cacheInode(dirIno) } file := &File{fs: f, inodeNum: childIno.InodeNum} @@ -885,12 +885,12 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f log.Printf("[DEBUG] Mkdir %q: updating root inode failed: %v", name, err) // The child directory was created successfully, just log the error } else { - // Update cache with parent directory inode - f.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if f.cacheMgr != nil { f.cacheMgr.BroadcastInvalidateInode(uint32(inode.InodeRoot), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + f.cacheInode(dirIno) } childDir := &Dir{fs: f, inodeNum: childIno.InodeNum} @@ -1430,12 +1430,12 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod log.Printf("[DEBUG] Create %q: updating parent inode %d failed: %v", name, d.inodeNum, err) // The file was created successfully, just log the error } else { - // Update cache with parent directory inode - d.fs.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if d.fs.cacheMgr != nil { d.fs.cacheMgr.BroadcastInvalidateInode(uint32(d.inodeNum), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + d.fs.cacheInode(dirIno) } // Create the file node @@ -1653,12 +1653,12 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out * // The child directory was created successfully, just log the error // and skip updating parent LinkCount/Mtime } else { - // Update cache with parent directory inode - d.fs.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if d.fs.cacheMgr != nil { d.fs.cacheMgr.BroadcastInvalidateInode(uint32(d.inodeNum), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + d.fs.cacheInode(dirIno) } // Create the directory node @@ -2708,15 +2708,15 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseCopied(cowDone) - // Cache the freshly-written inode for subsequent reads - f.fs.cacheInode(ino) - // Broadcast inode invalidation to other nodes for immediate visibility. // This is fire-and-forget - doesn't block the write operation. if f.fs.cacheMgr != nil { f.fs.cacheMgr.BroadcastInvalidateInode(uint32(f.inodeNum), ino.DataGeneration, ino.MetaGeneration) } + // Cache the freshly-written inode for subsequent reads + f.fs.cacheInode(ino) + // === WAL: Mark intent complete === // Now that all metadata and data have been persisted, mark the intent as complete. // On recovery, this prevents replaying already-completed operations. From 05a66661e055e843cac01b6b3f95fcee5598db11 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 16:11:41 -0500 Subject: [PATCH 4/6] test: wait out the kernel attribute cache in Docker coherency tests Co-Authored-By: Claude Opus 5.5 --- internal/cache/extent_cache_test.go | 17 +++++++++++++++++ tests/docker/scripts/run-tests.sh | 4 ++++ tests/test-manifest.yaml | 7 +++++++ 3 files changed, 28 insertions(+) diff --git a/internal/cache/extent_cache_test.go b/internal/cache/extent_cache_test.go index 0d5ccd7d..99f3b91e 100644 --- a/internal/cache/extent_cache_test.go +++ b/internal/cache/extent_cache_test.go @@ -36,3 +36,20 @@ func TestInvalidateExtentsForInodeDropsCachedInode(t *testing.T) { t.Error("inode 3 still cached after InvalidateExtentsForInode") } } + +func TestGetExtentCountsStaleEntryAsMiss(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + mgr.CacheExtent(&inode.Inode{InodeNum: 3, MetaGeneration: 4}, 0, 100) + mgr.GetExtent(&inode.Inode{InodeNum: 3, MetaGeneration: 5}, 0) + + if _, hits, misses, _ := mgr.ExtentCacheStats(); hits != 0 || misses != 1 { + t.Errorf("stale lookup counted hits=%d misses=%d, want 0 and 1", hits, misses) + } + if size, _, _, _ := mgr.ExtentCacheStats(); size != 0 { + t.Errorf("stale entry still cached, size=%d", size) + } +} diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index a25de0b6..0dc5d135 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2304,6 +2304,8 @@ test_inode_reuse_stale_read() { fi exec_on_node "$writer" sh -c "head -c 204800 /dev/zero | tr '\\0' p > $dir/reuse_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$new" + # The healthcheck's ls can leave a pre-write size in the reader's 1s kernel attribute cache. + sleep 1.5 local want got want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$new") got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$new") @@ -2337,6 +2339,8 @@ test_extent_cache_coherency() { # The padding file takes the freed block, so the rewrite lands in another one. exec_on_node "$writer" sh -c "truncate -s 0 $dir/$trunc && head -c 307200 /dev/zero | tr '\\0' p > $dir/extent_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$trunc" exec_on_node "$writer" sh -c "head -c 4096 /dev/zero | tr '\\0' b | dd of=$dir/$holes bs=1M seek=1 conv=notrunc 2>/dev/null" + # The healthcheck's ls can leave a mid-rewrite size in the reader's 1s kernel attribute cache. + sleep 1.5 local f want got failed=0 for f in "$trunc" "$holes"; do diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 91c953ab..fd056230 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1199,6 +1199,13 @@ scenarios: environments: go: internal/cache/extent_cache_test.go::TestInvalidateExtentsForInodeDropsCachedInode + - id: extent-cache-stale-miss + name: Stale extent lookups count as misses + description: A cached mapping for an older inode version is removed and counted as a miss, not a hit + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestGetExtentCountsStaleEntryAsMiss + - id: broadcast-ignore-own name: Broadcast ignores own messages description: Broadcaster ignores messages from itself From 41dca5e5de7f6ddaa3206352abc5061a5a45d821 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 16:13:58 -0500 Subject: [PATCH 5/6] feat(mount): add -attr-timeout for the kernel attribute and entry cache The default stays 1s. Document the cross-node staleness window it allows and the cost of 0, and fix the fs.go comments that said kernel caching was disabled: go-fuse replaces a zero AttrValid/EntryValid with this timeout. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + cmd/mount/main.go | 8 +++----- docs/API-CONFIG.md | 5 ++++- internal/goclusterfs/fs.go | 13 +++++-------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7f3b05ce..68a589ac 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ A VMFS-like clustered filesystem written in Go. GoClusterFS enables multiple nod - **Linux only.** GoClusterFS needs FUSE and O_DIRECT. - **Every file with data uses at least one full 1 MB block.** The block size is 1 MB, and small-file packing (SFB) is disabled since [#12](https://github.com/SupportTools/GoClusterFS/pull/12), so `-sfb` has no effect. - **Name lookups are slow.** Each lookup reads the whole directory from disk and scans it entry by entry. In one test on a single node, a lookup took about 25 ms, and `ls -l` on a 1,000-file directory took about 47 seconds. +- **Other nodes' changes can show up late.** The kernel caches file attributes and directory entries for `-attr-timeout` (default 1s). For that long, `stat` and `ls` can show another node's file with its old size and mtime, and a read can stop at the old size. A file that stays open can use the old size and block mappings for up to `-cache-ttl` (default 5s). Opening the file after `-attr-timeout` has passed reads the current data. See [Cache Flags](docs/API-CONFIG.md#cache-flags). - **On-disk format version 2** since [#15](https://github.com/SupportTools/GoClusterFS/pull/15). Mount refuses images formatted by earlier builds. Copy the data off with the old build, reformat, and copy it back. - **Upgrade all nodes at once.** Unmount on every node, upgrade, then mount again. Mixing old and new binaries on one device can serve stale data or make files unreadable. See the upgrade notes in [CHANGELOG.md](CHANGELOG.md). - **Snapshots and clones are experimental.** Before you mount an image that has snapshots or clones with a newer build, read the upgrade notes in [CHANGELOG.md](CHANGELOG.md). They say whether to run `goclusterfs-fsck` first. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index a73fd483..87f9e1c6 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -41,6 +41,7 @@ func main() { // Cache configuration cacheStrict = flag.Bool("cache-strict", false, "Enable strict cache mode (always validate)") cacheTTL = flag.Duration("cache-ttl", 5*time.Second, "Cache TTL before validation") + attrTimeout = flag.Duration("attr-timeout", time.Second, "How long the kernel caches file attributes and directory entries") // Journal configuration journalEnabled = flag.Bool("journal", true, "Enable write-ahead journal for crash consistency") @@ -321,9 +322,6 @@ func main() { // Configure go-fuse mount options // AttrTimeout and EntryTimeout control kernel-side caching - attrTimeout := 1 * time.Second - entryTimeout := 1 * time.Second - mountOpts := &fs.Options{ MountOptions: fuse.MountOptions{ FsName: "goclusterfs", @@ -333,8 +331,8 @@ func main() { DisableXAttrs: false, // Note: AllowOther requires 'user_allow_other' in /etc/fuse.conf }, - AttrTimeout: &attrTimeout, - EntryTimeout: &entryTimeout, + AttrTimeout: attrTimeout, + EntryTimeout: attrTimeout, } // Mount filesystem using go-fuse v2 API diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 2e5b1945..26190947 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -42,8 +42,11 @@ These flags are passed when starting `goclusterfs`: |------|------|---------|-------------| | `-cache-strict` | bool | `false` | Enable strict cache mode (always validate against disk) | | `-cache-ttl` | duration | `5s` | Cache entry TTL before revalidation required | +| `-attr-timeout` | duration | `1s` | How long the kernel caches file attributes and directory entries | -The kernel caches file attributes and directory entries for 1s (the FUSE attribute and entry timeouts). For up to 1s after another node changes a file, `stat` and path lookups on this node can return the old size, and a read can stop at the old size even right after `open`. After that, opening a file reads its inode from disk, so the reader sees the other node's completed writes. Reads through a file that is already open use the inode cache and can use the old size and block mappings for up to `-cache-ttl`, including a block that another node has since freed and reused for other data. With broadcast enabled, writes and directory changes invalidate peers' caches sooner; truncate, fallocate and unlink do not. +The kernel caches file attributes and directory entries for `-attr-timeout`. For that long after another node changes a file, `stat` and path lookups on this node can return the old size, and a read can stop at the old size even right after `open`. After that, opening a file reads its inode from disk, so the reader sees the other node's completed writes. Reads through a file that is already open use the inode cache and can use the old size and block mappings for up to `-cache-ttl`, including a block that another node has since freed and reused for other data. With broadcast enabled, writes and directory changes invalidate peers' caches sooner; truncate, fallocate and unlink do not. + +`-attr-timeout=0` makes `stat` and path lookups always current, but each `stat` and path step then reads the directory from disk under a cluster lock. In one test on a single mount, 2000 `stat` calls on one file took 13.9s at 0 and 0.01s at 1s, and `ls -l` of 300 files took 11.8s and 3.7s. ### Journal Flags diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 81274c49..87e5c32d 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -484,7 +484,7 @@ func (f *FS) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) s return syscall.EIO } - // Disable FUSE kernel caching for cluster operation. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) @@ -533,7 +533,7 @@ func (f *FS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.I return nil, syscall.EIO } - // Disable FUSE kernel caching for cluster operation. + // Zero means go-fuse applies the mount's -attr-timeout. out.EntryValid = 0 out.AttrValid = 0 @@ -1118,8 +1118,7 @@ func (d *Dir) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) return syscall.EIO } - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify attributes at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) @@ -1168,8 +1167,7 @@ func (d *Dir) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs. return nil, syscall.EIO } - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify the filesystem at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.EntryValid = 0 out.AttrValid = 0 @@ -1951,8 +1949,7 @@ func (f *File) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) log.Printf("[DEBUG] File.Getattr: inode=%d Size=%d AllocatedBlocks=%d", f.inodeNum, ino.Size, ino.AllocatedBlocks) - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify attributes at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) From c44147f4fbb9c0deae0cdf0556506fbe6c42223b Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 16:13:58 -0500 Subject: [PATCH 6/6] test: check that -attr-timeout=0 shows remote size changes at once Co-Authored-By: Claude Opus 5.5 --- tests/integration/multinode_test.go | 31 +++++++++++++++++++++++++++++ tests/test-manifest.yaml | 10 ++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index 8d4a2d0f..a95a7a96 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -1181,6 +1181,37 @@ func TestMultiNodeExtentCacheCoherency(t *testing.T) { }) } +// TestMultiNodeAttrTimeoutZero checks that with -attr-timeout=0 a node's next stat +// shows another node's size change, with no kernel attribute cache delay. +func TestMultiNodeAttrTimeoutZero(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + cluster.Node(1).ExtraArgs = []string{"-broadcast=false", "-attr-timeout=0"} + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + const name = "grow.txt" + writeOrFail(t, writer, name, []byte("0123456789"), 0) + expectSize(t, reader, name, 10) + writeOrFail(t, writer, name, []byte("abcdefghij"), 10) + expectSize(t, reader, name, 20) +} + +func expectSize(t *testing.T, m *MountedFS, name string, want int64) { + t.Helper() + fi, err := m.Stat(name) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if fi.Size() != want { + t.Errorf("stat %s: size %d, want %d", name, fi.Size(), want) + } +} + func writeOrFail(t *testing.T, m *MountedFS, name string, data []byte, offset int64) { t.Helper() if err := m.WriteFileAt(name, data, offset); err != nil { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index fd056230..c7c2b7f1 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -378,6 +378,16 @@ scenarios: docker: run-tests.sh::test_extent_cache_coherency vm: run-vm-tests.sh::test_extent_cache_coherency + # The Docker and VM harnesses have no per-node mount flags, so they cannot run one node with -attr-timeout=0. + - id: multinode-attr-timeout-zero + name: Attribute timeout 0 shows remote size changes + description: A node mounted with -attr-timeout=0 sees another node's size change on its next stat + priority: medium + environments: + go: multinode_test.go::TestMultiNodeAttrTimeoutZero + docker: TODO + vm: TODO + # =========================================================================== # NETWORK AND RESILIENCE TESTS # ===========================================================================