diff --git a/README.md b/README.md index 7f3b05ce..68a589ac 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ A VMFS-like clustered filesystem written in Go. GoClusterFS enables multiple nod - **Linux only.** GoClusterFS needs FUSE and O_DIRECT. - **Every file with data uses at least one full 1 MB block.** The block size is 1 MB, and small-file packing (SFB) is disabled since [#12](https://github.com/SupportTools/GoClusterFS/pull/12), so `-sfb` has no effect. - **Name lookups are slow.** Each lookup reads the whole directory from disk and scans it entry by entry. In one test on a single node, a lookup took about 25 ms, and `ls -l` on a 1,000-file directory took about 47 seconds. +- **Other nodes' changes can show up late.** The kernel caches file attributes and directory entries for `-attr-timeout` (default 1s). For that long, `stat` and `ls` can show another node's file with its old size and mtime, and a read can stop at the old size. A file that stays open can use the old size and block mappings for up to `-cache-ttl` (default 5s). Opening the file after `-attr-timeout` has passed reads the current data. See [Cache Flags](docs/API-CONFIG.md#cache-flags). - **On-disk format version 2** since [#15](https://github.com/SupportTools/GoClusterFS/pull/15). Mount refuses images formatted by earlier builds. Copy the data off with the old build, reformat, and copy it back. - **Upgrade all nodes at once.** Unmount on every node, upgrade, then mount again. Mixing old and new binaries on one device can serve stale data or make files unreadable. See the upgrade notes in [CHANGELOG.md](CHANGELOG.md). - **Snapshots and clones are experimental.** Before you mount an image that has snapshots or clones with a newer build, read the upgrade notes in [CHANGELOG.md](CHANGELOG.md). They say whether to run `goclusterfs-fsck` first. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index a73fd483..87f9e1c6 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -41,6 +41,7 @@ func main() { // Cache configuration cacheStrict = flag.Bool("cache-strict", false, "Enable strict cache mode (always validate)") cacheTTL = flag.Duration("cache-ttl", 5*time.Second, "Cache TTL before validation") + attrTimeout = flag.Duration("attr-timeout", time.Second, "How long the kernel caches file attributes and directory entries") // Journal configuration journalEnabled = flag.Bool("journal", true, "Enable write-ahead journal for crash consistency") @@ -321,9 +322,6 @@ func main() { // Configure go-fuse mount options // AttrTimeout and EntryTimeout control kernel-side caching - attrTimeout := 1 * time.Second - entryTimeout := 1 * time.Second - mountOpts := &fs.Options{ MountOptions: fuse.MountOptions{ FsName: "goclusterfs", @@ -333,8 +331,8 @@ func main() { DisableXAttrs: false, // Note: AllowOther requires 'user_allow_other' in /etc/fuse.conf }, - AttrTimeout: &attrTimeout, - EntryTimeout: &entryTimeout, + AttrTimeout: attrTimeout, + EntryTimeout: attrTimeout, } // Mount filesystem using go-fuse v2 API diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 8262fcef..26190947 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -42,6 +42,11 @@ These flags are passed when starting `goclusterfs`: |------|------|---------|-------------| | `-cache-strict` | bool | `false` | Enable strict cache mode (always validate against disk) | | `-cache-ttl` | duration | `5s` | Cache entry TTL before revalidation required | +| `-attr-timeout` | duration | `1s` | How long the kernel caches file attributes and directory entries | + +The kernel caches file attributes and directory entries for `-attr-timeout`. For that long after another node changes a file, `stat` and path lookups on this node can return the old size, and a read can stop at the old size even right after `open`. After that, opening a file reads its inode from disk, so the reader sees the other node's completed writes. Reads through a file that is already open use the inode cache and can use the old size and block mappings for up to `-cache-ttl`, including a block that another node has since freed and reused for other data. With broadcast enabled, writes and directory changes invalidate peers' caches sooner; truncate, fallocate and unlink do not. + +`-attr-timeout=0` makes `stat` and path lookups always current, but each `stat` and path step then reads the directory from disk under a cluster lock. In one test on a single mount, 2000 `stat` calls on one file took 13.9s at 0 and 0.01s at 1s, and `ls -l` of 300 files took 11.8s and 3.7s. ### Journal Flags @@ -143,7 +148,7 @@ Time after which a cached inode must be revalidated against disk. | Memory | ~3.2 MB (for defaults) | | Recommended | 50,000 - 500,000 | -The maximum number of extent mappings to cache. Extent cache maps (inode, logical block) → physical block. +The maximum number of extent mappings to cache. Extent cache maps (inode, logical block) → physical block. A cached mapping is only used with the inode version it was looked up with, so any inode write by any node makes it stale. **Sizing guidance**: - Large files benefit more from extent caching diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 2df642d5..6f61164d 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -36,6 +36,15 @@ func (c *CachedInode) IsStale(diskDataGen, diskMetaGen uint64) bool { return c.DataGeneration != diskDataGen || c.MetaGeneration != diskMetaGen } +// newerThan reports whether c holds a later version of the inode than in. +// DataGeneration only changes when the inode number is reused, so it orders first. +func (c *CachedInode) newerThan(in *inode.Inode) bool { + if c.DataGeneration != in.DataGeneration { + return c.DataGeneration > in.DataGeneration + } + return c.MetaGeneration > in.MetaGeneration +} + // NeedsValidation returns true if the cache entry needs revalidation against disk. func (c *CachedInode) NeedsValidation(ttl time.Duration, strictMode bool) bool { if strictMode { @@ -254,7 +263,8 @@ func (m *Manager) BroadcastEnabled() bool { } // BroadcastInvalidateInode invalidates an inode locally AND broadcasts -// the invalidation to peer nodes. Call this after modifying an inode. +// the invalidation to peer nodes. Call this after modifying an inode, and +// cache the new inode after this call, since it drops the local entry. // // Parameters: // - inodeNum: The inode number being invalidated diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 52d27f06..2591a4ed 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -381,16 +381,17 @@ func TestCachedExtent(t *testing.T) { cached := &CachedExtent{ PhysicalBlock: 100, InodeDataGen: 5, + InodeMetaGen: 7, } - // Same generation - not stale - if cached.IsStale(5) { - t.Error("IsStale() should return false when generation matches") + if cached.IsStale(5, 7) { + t.Error("IsStale() should return false when generations match") } - - // Different generation - stale - if !cached.IsStale(6) { - t.Error("IsStale() should return true when generation differs") + if !cached.IsStale(6, 7) { + t.Error("IsStale() should return true when DataGeneration differs") + } + if !cached.IsStale(5, 8) { + t.Error("IsStale() should return true when MetaGeneration differs") } }) } @@ -491,14 +492,14 @@ func TestExtentCache(t *testing.T) { t.Fatal("Get() should find cached extent") } - // Same generation - not stale - if got.IsStale(10) { - t.Error("Extent should not be stale with matching generation") + if got.IsStale(10, 0) { + t.Error("Extent should not be stale with matching generations") } - - // Different generation - stale - if !got.IsStale(11) { - t.Error("Extent should be stale with different generation") + if !got.IsStale(11, 0) { + t.Error("Extent should be stale with different DataGeneration") + } + if !got.IsStale(10, 1) { + t.Error("Extent should be stale with different MetaGeneration") } }) diff --git a/internal/cache/extent_cache.go b/internal/cache/extent_cache.go index 698595ea..b1754f74 100644 --- a/internal/cache/extent_cache.go +++ b/internal/cache/extent_cache.go @@ -16,122 +16,92 @@ type ExtentCacheKey struct { // CachedExtent holds a cached extent lookup result. type CachedExtent struct { - // PhysicalBlock is the physical disk block number. - // A value of 0 means "hole" (sparse file - unallocated region). + // PhysicalBlock is the physical disk block number, or extent.Hole. PhysicalBlock uint64 - // InodeDataGen is the inode's DataGeneration at cache time. - // When the inode's DataGeneration changes, all extent cache - // entries for that inode become stale. + // InodeDataGen and InodeMetaGen are the inode's generations at cache time. + // Every extent mapping change is committed by an inode write, which bumps + // MetaGeneration, so a mapping is only valid for the inode version it was + // looked up with. InodeDataGen uint64 + InodeMetaGen uint64 // CachedAt is when this entry was cached. CachedAt time.Time } -// IsStale returns true if the cached extent is stale relative to the inode. -func (c *CachedExtent) IsStale(currentDataGen uint64) bool { - return c.InodeDataGen != currentDataGen +// IsStale returns true if the cached extent was looked up with a different inode version. +func (c *CachedExtent) IsStale(dataGen, metaGen uint64) bool { + return c.InodeDataGen != dataGen || c.InodeMetaGen != metaGen } -// GetExtent retrieves a cached extent mapping. -// -// The retrieval flow is: -// 1. Check extent cache for (inodeNum, logicalBlock) -// 2. If found, validate against inode's DataGeneration -// 3. If valid, return cached physical block -// 4. If stale or not found, return cache miss +// GetExtent retrieves a cached extent mapping for this version of ino. // // This function does NOT perform disk I/O - caller must fall back to // extent.Manager.LookupPhysical() on cache miss. // // Returns: -// - physicalBlock: The physical block number (0 = hole) +// - physicalBlock: The physical block number or extent.Hole // - found: true if cache hit and valid -func (m *Manager) GetExtent(inodeNum uint32, logicalBlock uint64, inodeDataGen uint64) (physicalBlock uint64, found bool) { +func (m *Manager) GetExtent(ino *inode.Inode, logicalBlock uint64) (physicalBlock uint64, found bool) { key := ExtentCacheKey{ - InodeNum: inodeNum, + InodeNum: ino.InodeNum, LogicalBlock: logicalBlock, } - cached, ok := m.extents.Get(key) + cached, ok := m.extents.GetIf(key, func(c *CachedExtent) bool { + return !c.IsStale(ino.DataGeneration, ino.MetaGeneration) + }) if !ok { return 0, false } - - // Validate against inode's current DataGeneration - if cached.IsStale(inodeDataGen) { - // Stale entry - remove it - m.extents.Delete(key) - return 0, false - } - return cached.PhysicalBlock, true } -// CacheExtent stores an extent mapping in the cache. +// CacheExtent stores an extent mapping looked up with this version of ino. // // Call this after a successful extent.Manager.LookupPhysical() to cache -// the result for future reads. -// -// Parameters: -// - inodeNum: The inode number -// - logicalBlock: The logical file block being looked up -// - physicalBlock: The physical disk block (0 for hole) -// - inodeDataGen: The inode's current DataGeneration -func (m *Manager) CacheExtent(inodeNum uint32, logicalBlock uint64, physicalBlock uint64, inodeDataGen uint64) { +// the result for future reads. physicalBlock is extent.Hole for a hole. +func (m *Manager) CacheExtent(ino *inode.Inode, logicalBlock uint64, physicalBlock uint64) { if m.config.ExtentCacheSize == 0 { return // Extent cache disabled } key := ExtentCacheKey{ - InodeNum: inodeNum, + InodeNum: ino.InodeNum, LogicalBlock: logicalBlock, } cached := &CachedExtent{ PhysicalBlock: physicalBlock, - InodeDataGen: inodeDataGen, + InodeDataGen: ino.DataGeneration, + InodeMetaGen: ino.MetaGeneration, CachedAt: time.Now(), } m.extents.Put(key, cached) } -// InvalidateExtents removes all cached extents for an inode. +// InvalidateExtents makes cached extents for an inode stale. // -// Call this when: -// - File is truncated (all extents may change) -// - File data is modified in a way that changes extent mapping -// - Inode's DataGeneration has changed -// -// Note: Individual extent invalidation is handled lazily via generation -// check in GetExtent. This function is for bulk invalidation. +// Call this when another node changed the inode's extent mapping (truncate, +// hole fill, CoW). It drops the cached inode, so the next lookup reads the +// current inode version from disk and every mapping cached for an older +// version misses. func (m *Manager) InvalidateExtents(inodeNum uint32) { - // Unfortunately, our LRU doesn't support prefix/range deletion. - // We rely on lazy invalidation via DataGeneration checks. - // For bulk invalidation, we'd need to iterate all entries. - // - // For now, extents are invalidated lazily when GetExtent detects - // a DataGeneration mismatch. This is acceptable because: - // 1. Stale entries don't return incorrect data (generation check) - // 2. They get evicted naturally by LRU or replaced on next lookup - // 3. InvalidateInodes() already handles the common case + m.inodes.Delete(inodeNum) } -// InvalidateExtentsForInodes removes cached extents when inode generation changes. -// This is called from OnLockAcquire for extent-level locks. +// InvalidateExtentsForInodes makes cached extents for several inodes stale. func (m *Manager) InvalidateExtentsForInodes(inodeNums ...uint32) { - // Lazy invalidation via generation check handles this. - // See InvalidateExtents() comment. + for _, inodeNum := range inodeNums { + m.InvalidateExtents(inodeNum) + } } -// InvalidateExtentsForInode removes cached extents for a single inode. +// InvalidateExtentsForInode makes cached extents for a single inode stale. // This satisfies the broadcast.InvalidationHandler interface. func (m *Manager) InvalidateExtentsForInode(inodeNum uint32) { - // Lazy invalidation via DataGeneration check handles this. - // When the next GetExtent() call is made, the generation mismatch - // will cause the stale entry to be evicted. m.InvalidateExtents(inodeNum) } @@ -142,26 +112,24 @@ func (m *Manager) InvalidateExtentsForInode(inodeNum uint32) { // // Parameters: // - extMgr: The extent manager for disk fallback -// - ino: The inode (must have current DataGeneration) +// - ino: A committed inode version (no uncommitted extent edits) // - logicalBlock: The logical file block to look up // // Returns: -// - physicalBlock: The physical disk block (0 = hole) +// - physicalBlock: The physical disk block or extent.Hole // - err: Any error from disk lookup func (m *Manager) LookupPhysicalCached(extMgr *extent.Manager, ino *inode.Inode, logicalBlock uint64) (uint64, error) { - // Try cache first - if phys, found := m.GetExtent(ino.InodeNum, logicalBlock, ino.DataGeneration); found { + if phys, found := m.GetExtent(ino, logicalBlock); found { return phys, nil } - // Cache miss - look up from disk phys, err := extMgr.LookupPhysical(ino, logicalBlock) if err != nil { return 0, err } // Cache the result (including holes, which map to extent.Hole) - m.CacheExtent(ino.InodeNum, logicalBlock, phys, ino.DataGeneration) + m.CacheExtent(ino, logicalBlock, phys) return phys, nil } diff --git a/internal/cache/extent_cache_test.go b/internal/cache/extent_cache_test.go new file mode 100644 index 00000000..99f3b91e --- /dev/null +++ b/internal/cache/extent_cache_test.go @@ -0,0 +1,55 @@ +package cache + +import ( + "testing" + + "github.com/supporttools/GoClusterFS/internal/inode" +) + +func TestGetExtentMissesAfterInodeWrite(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + v1 := &inode.Inode{InodeNum: 3, DataGeneration: 1, MetaGeneration: 4} + mgr.CacheExtent(v1, 0, 100) + if phys, ok := mgr.GetExtent(v1, 0); !ok || phys != 100 { + t.Fatalf("GetExtent with the caching version = (%d, %v), want (100, true)", phys, ok) + } + + v2 := &inode.Inode{InodeNum: 3, DataGeneration: 1, MetaGeneration: 5} + if phys, ok := mgr.GetExtent(v2, 0); ok { + t.Errorf("GetExtent after an inode write hit with block %d, want a miss", phys) + } +} + +func TestInvalidateExtentsForInodeDropsCachedInode(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + mgr.CacheInode(&inode.Inode{InodeNum: 3, MetaGeneration: 1}) + mgr.InvalidateExtentsForInode(3) + if mgr.IsCached(3) { + t.Error("inode 3 still cached after InvalidateExtentsForInode") + } +} + +func TestGetExtentCountsStaleEntryAsMiss(t *testing.T) { + mgr, err := NewManager(DefaultConfig(), nil, nil) + if err != nil { + t.Fatal(err) + } + + mgr.CacheExtent(&inode.Inode{InodeNum: 3, MetaGeneration: 4}, 0, 100) + mgr.GetExtent(&inode.Inode{InodeNum: 3, MetaGeneration: 5}, 0) + + if _, hits, misses, _ := mgr.ExtentCacheStats(); hits != 0 || misses != 1 { + t.Errorf("stale lookup counted hits=%d misses=%d, want 0 and 1", hits, misses) + } + if size, _, _, _ := mgr.ExtentCacheStats(); size != 0 { + t.Errorf("stale entry still cached, size=%d", size) + } +} diff --git a/internal/cache/inode_cache.go b/internal/cache/inode_cache.go index 7d075d5c..276aafc6 100644 --- a/internal/cache/inode_cache.go +++ b/internal/cache/inode_cache.go @@ -100,13 +100,14 @@ func (m *Manager) InvalidateAll() { } // CacheInode explicitly caches an inode. -// Use this after reading an inode from disk in a write path, -// so subsequent reads can benefit from the cached data. +// Use this after reading or writing an inode, so subsequent reads can +// benefit from the cached data. A newer cached copy is kept. func (m *Manager) CacheInode(in *inode.Inode) { m.cacheInode(in) } -// cacheInode adds an inode to the cache (internal helper). +// cacheInode adds an inode to the cache unless a newer copy is already cached. +// A disk read can finish after a local write cached a later version. func (m *Manager) cacheInode(in *inode.Inode) { now := time.Now() cached := &CachedInode{ @@ -116,7 +117,7 @@ func (m *Manager) cacheInode(in *inode.Inode) { CachedAt: now, LastValidated: now, } - m.inodes.Put(in.InodeNum, cached) + m.inodes.PutIf(in.InodeNum, cached, func(old *CachedInode) bool { return !old.newerThan(in) }) m.negatives.Delete(in.InodeNum) } diff --git a/internal/cache/inode_cache_test.go b/internal/cache/inode_cache_test.go index 7839851f..dfb007d7 100644 --- a/internal/cache/inode_cache_test.go +++ b/internal/cache/inode_cache_test.go @@ -108,3 +108,36 @@ func TestCacheInodeClearsNegativeEntry(t *testing.T) { t.Errorf("GetInode returned Size %d, want 5", got.Size) } } + +func TestCacheInodeKeepsNewerCopy(t *testing.T) { + tbl := newTestInodeTable(t) + mgr, err := NewManager(DefaultConfig(), tbl, nil) + if err != nil { + t.Fatal(err) + } + + claimWithSize(t, tbl, inode.InodeFirstUsable, 3) + older, err := tbl.Read(inode.InodeFirstUsable) + if err != nil { + t.Fatal(err) + } + newer, err := tbl.Read(inode.InodeFirstUsable) + if err != nil { + t.Fatal(err) + } + newer.Size = 10 + if err := tbl.Write(context.Background(), newer); err != nil { + t.Fatal(err) + } + + mgr.CacheInode(newer) + mgr.CacheInode(older) + + got, err := mgr.GetInode(inode.InodeFirstUsable) + if err != nil { + t.Fatalf("GetInode: %v", err) + } + if got.Size != 10 { + t.Errorf("GetInode returned Size %d from the older copy, want 10", got.Size) + } +} diff --git a/internal/cache/lru.go b/internal/cache/lru.go index e0f65c24..58bce5af 100644 --- a/internal/cache/lru.go +++ b/internal/cache/lru.go @@ -64,6 +64,28 @@ func (l *LRU[K, V]) Get(key K) (V, bool) { return zero, false } +// GetIf is Get for entries that can go stale: an entry for which valid +// returns false is removed and counted as a miss. +func (l *LRU[K, V]) GetIf(key K, valid func(V) bool) (V, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + if elem, exists := l.items[key]; exists { + entry := elem.Value.(*lruEntry[K, V]) + if valid(entry.value) { + l.order.MoveToFront(elem) + l.hits++ + return entry.value, true + } + l.order.Remove(elem) + delete(l.items, key) + } + + var zero V + l.misses++ + return zero, false +} + // Peek retrieves a value without affecting LRU ordering. // Use this for validation checks that shouldn't promote entries. func (l *LRU[K, V]) Peek(key K) (V, bool) { @@ -85,6 +107,23 @@ func (l *LRU[K, V]) Put(key K, value V) (evictedKey K, evictedValue V, evicted b l.mu.Lock() defer l.mu.Unlock() + return l.putLocked(key, value) +} + +// PutIf stores value unless key is already present and replace(old) returns false. +// The check and the store are atomic. Returns whether value was stored. +func (l *LRU[K, V]) PutIf(key K, value V, replace func(old V) bool) bool { + l.mu.Lock() + defer l.mu.Unlock() + + if elem, exists := l.items[key]; exists && !replace(elem.Value.(*lruEntry[K, V]).value) { + return false + } + l.putLocked(key, value) + return true +} + +func (l *LRU[K, V]) putLocked(key K, value V) (evictedKey K, evictedValue V, evicted bool) { if l.capacity == 0 { return } diff --git a/internal/cache/sharded_lru.go b/internal/cache/sharded_lru.go index e13303d0..53b96655 100644 --- a/internal/cache/sharded_lru.go +++ b/internal/cache/sharded_lru.go @@ -56,6 +56,11 @@ func (s *ShardedLRU[K, V]) Get(key K) (V, bool) { return s.shards[s.shard(key)].Get(key) } +// GetIf retrieves a value if valid accepts it; otherwise the entry is removed. +func (s *ShardedLRU[K, V]) GetIf(key K, valid func(V) bool) (V, bool) { + return s.shards[s.shard(key)].GetIf(key, valid) +} + // Peek retrieves a value without affecting LRU ordering. func (s *ShardedLRU[K, V]) Peek(key K) (V, bool) { return s.shards[s.shard(key)].Peek(key) @@ -67,6 +72,11 @@ func (s *ShardedLRU[K, V]) Put(key K, value V) (evictedKey K, evictedValue V, ev return s.shards[s.shard(key)].Put(key, value) } +// PutIf stores value unless key is already present and replace(old) returns false. +func (s *ShardedLRU[K, V]) PutIf(key K, value V, replace func(old V) bool) bool { + return s.shards[s.shard(key)].PutIf(key, value, replace) +} + // Delete removes an entry from the cache. func (s *ShardedLRU[K, V]) Delete(key K) (V, bool) { return s.shards[s.shard(key)].Delete(key) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 140f756d..87e5c32d 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -484,7 +484,7 @@ func (f *FS) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) s return syscall.EIO } - // Disable FUSE kernel caching for cluster operation. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) @@ -533,7 +533,7 @@ func (f *FS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.I return nil, syscall.EIO } - // Disable FUSE kernel caching for cluster operation. + // Zero means go-fuse applies the mount's -attr-timeout. out.EntryValid = 0 out.AttrValid = 0 @@ -724,12 +724,12 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, log.Printf("[DEBUG] Create %q: updating root inode failed: %v", name, err) // The file was created successfully, just log the error } else { - // Update cache with parent directory inode - f.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if f.cacheMgr != nil { f.cacheMgr.BroadcastInvalidateInode(uint32(inode.InodeRoot), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + f.cacheInode(dirIno) } file := &File{fs: f, inodeNum: childIno.InodeNum} @@ -885,12 +885,12 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f log.Printf("[DEBUG] Mkdir %q: updating root inode failed: %v", name, err) // The child directory was created successfully, just log the error } else { - // Update cache with parent directory inode - f.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if f.cacheMgr != nil { f.cacheMgr.BroadcastInvalidateInode(uint32(inode.InodeRoot), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + f.cacheInode(dirIno) } childDir := &Dir{fs: f, inodeNum: childIno.InodeNum} @@ -1118,8 +1118,7 @@ func (d *Dir) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) return syscall.EIO } - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify attributes at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) @@ -1168,8 +1167,7 @@ func (d *Dir) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs. return nil, syscall.EIO } - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify the filesystem at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.EntryValid = 0 out.AttrValid = 0 @@ -1430,12 +1428,12 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod log.Printf("[DEBUG] Create %q: updating parent inode %d failed: %v", name, d.inodeNum, err) // The file was created successfully, just log the error } else { - // Update cache with parent directory inode - d.fs.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if d.fs.cacheMgr != nil { d.fs.cacheMgr.BroadcastInvalidateInode(uint32(d.inodeNum), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + d.fs.cacheInode(dirIno) } // Create the file node @@ -1653,12 +1651,12 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out * // The child directory was created successfully, just log the error // and skip updating parent LinkCount/Mtime } else { - // Update cache with parent directory inode - d.fs.cacheInode(dirIno) // Broadcast invalidation so other nodes refresh their directory cache if d.fs.cacheMgr != nil { d.fs.cacheMgr.BroadcastInvalidateInode(uint32(d.inodeNum), dirIno.DataGeneration, dirIno.MetaGeneration) } + // Update cache with parent directory inode + d.fs.cacheInode(dirIno) } // Create the directory node @@ -1951,8 +1949,7 @@ func (f *File) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) log.Printf("[DEBUG] File.Getattr: inode=%d Size=%d AllocatedBlocks=%d", f.inodeNum, ino.Size, ino.AllocatedBlocks) - // Disable FUSE kernel caching for cluster operation. - // Other nodes may modify attributes at any time. + // Zero means go-fuse applies the mount's -attr-timeout. out.AttrValid = 0 out.Ino = uint64(ino.InodeNum) @@ -2708,15 +2705,15 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseCopied(cowDone) - // Cache the freshly-written inode for subsequent reads - f.fs.cacheInode(ino) - // Broadcast inode invalidation to other nodes for immediate visibility. // This is fire-and-forget - doesn't block the write operation. if f.fs.cacheMgr != nil { f.fs.cacheMgr.BroadcastInvalidateInode(uint32(f.inodeNum), ino.DataGeneration, ino.MetaGeneration) } + // Cache the freshly-written inode for subsequent reads + f.fs.cacheInode(ino) + // === WAL: Mark intent complete === // Now that all metadata and data have been persisted, mark the intent as complete. // On recovery, this prevents replaying already-completed operations. diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 539ccc68..0dc5d135 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2320,6 +2320,46 @@ test_inode_reuse_stale_read() { return 0 } +# Test: Extent cache coherency across nodes +# Node 1 reads files, node 0 changes their block mappings (truncate and rewrite, +# hole fill), and node 1 must read the new data. +test_extent_cache_coherency() { + log "Testing extent cache coherency across nodes..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]}" + local dir=/mnt/goclusterfs + local trunc="extent_trunc_$(date +%s).bin" + local holes="extent_holes_$(date +%s).bin" + + exec_on_node "$writer" sh -c "head -c 307200 /dev/zero | tr '\\0' o > $dir/$trunc" + exec_on_node "$writer" sh -c "head -c 4096 /dev/zero | tr '\\0' a > $dir/$holes && head -c 4096 /dev/zero | tr '\\0' c | dd of=$dir/$holes bs=1M seek=2 conv=notrunc 2>/dev/null" + exec_on_node "$reader" sh -c "cat $dir/$trunc $dir/$holes > /dev/null" + + # The padding file takes the freed block, so the rewrite lands in another one. + exec_on_node "$writer" sh -c "truncate -s 0 $dir/$trunc && head -c 307200 /dev/zero | tr '\\0' p > $dir/extent_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$trunc" + exec_on_node "$writer" sh -c "head -c 4096 /dev/zero | tr '\\0' b | dd of=$dir/$holes bs=1M seek=1 conv=notrunc 2>/dev/null" + # The healthcheck's ls can leave a mid-rewrite size in the reader's 1s kernel attribute cache. + sleep 1.5 + + local f want got failed=0 + for f in "$trunc" "$holes"; do + want=$(exec_on_node "$writer" sh -c "md5sum < $dir/$f") + got=$(exec_on_node "$reader" sh -c "md5sum < $dir/$f") + if [[ "$got" != "$want" ]]; then + log "ERROR: $reader read stale data from $f: md5 '$got', want '$want'" + failed=1 + fi + done + exec_on_node "$writer" sh -c "rm -f $dir/$trunc $dir/$holes $dir/extent_pad.bin" + + if [[ $failed -ne 0 ]]; then + return 1 + fi + log "Extent cache coherency test passed" + return 0 +} + # Print the free 1MB blocks that statfs reports on node $1 statfs_free_blocks() { exec_on_node "$1" stat -f -c %f /mnt/goclusterfs @@ -3116,6 +3156,7 @@ run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidatio run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read +run_test "Extent Cache Coherency" test_extent_cache_coherency run_test "Statfs After Remount" test_statfs_after_remount run_test "Statfs Multi-Node" test_statfs_multinode run_test "Rolling Restart" test_rolling_restart diff --git a/tests/integration/multinode_test.go b/tests/integration/multinode_test.go index b6183ead..a95a7a96 100644 --- a/tests/integration/multinode_test.go +++ b/tests/integration/multinode_test.go @@ -6,6 +6,7 @@ import ( "bytes" "fmt" "math/rand" + "os" "sync" "syscall" "testing" @@ -1134,6 +1135,106 @@ func inodeNumber(t *testing.T, m *MountedFS, name string) uint64 { return fi.Sys().(*syscall.Stat_t).Ino } +// TestMultiNodeExtentCacheCoherency checks that a node which cached a file's block +// mappings reads the new data after another node changes those mappings. +func TestMultiNodeExtentCacheCoherency(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + cluster.Node(1).ExtraArgs = []string{"-broadcast=false"} + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + t.Run("TruncateRewrite", func(t *testing.T) { + const name = "trunc.bin" + writeOrFail(t, writer, name, bytes.Repeat([]byte("o"), 300<<10), 0) + expectContent(t, reader, name, bytes.Repeat([]byte("o"), 300<<10)) + + if err := os.Truncate(writer.Path(name), 0); err != nil { + t.Fatalf("Writer: truncate %s: %v", name, err) + } + // The padding file takes the freed block, so the rewrite lands in another one. + writeOrFail(t, writer, "trunc_pad.bin", bytes.Repeat([]byte("p"), 300<<10), 0) + writeOrFail(t, writer, name, bytes.Repeat([]byte("n"), 300<<10), 0) + + expectContent(t, reader, name, bytes.Repeat([]byte("n"), 300<<10)) + }) + + t.Run("HoleFill", func(t *testing.T) { + const name = "holes.bin" + const blockSize = 1 << 20 + chunk := func(c byte) []byte { return bytes.Repeat([]byte{c}, 4096) } + writeOrFail(t, writer, name, chunk('a'), 0) + writeOrFail(t, writer, name, chunk('c'), 2*blockSize) + + want := make([]byte, 2*blockSize+4096) + copy(want, chunk('a')) + copy(want[2*blockSize:], chunk('c')) + expectContent(t, reader, name, want) + + writeOrFail(t, writer, name, chunk('b'), blockSize) + copy(want[blockSize:], chunk('b')) + expectContent(t, reader, name, want) + }) +} + +// TestMultiNodeAttrTimeoutZero checks that with -attr-timeout=0 a node's next stat +// shows another node's size change, with no kernel attribute cache delay. +func TestMultiNodeAttrTimeoutZero(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + cluster.Node(1).ExtraArgs = []string{"-broadcast=false", "-attr-timeout=0"} + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + const name = "grow.txt" + writeOrFail(t, writer, name, []byte("0123456789"), 0) + expectSize(t, reader, name, 10) + writeOrFail(t, writer, name, []byte("abcdefghij"), 10) + expectSize(t, reader, name, 20) +} + +func expectSize(t *testing.T, m *MountedFS, name string, want int64) { + t.Helper() + fi, err := m.Stat(name) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if fi.Size() != want { + t.Errorf("stat %s: size %d, want %d", name, fi.Size(), want) + } +} + +func writeOrFail(t *testing.T, m *MountedFS, name string, data []byte, offset int64) { + t.Helper() + if err := m.WriteFileAt(name, data, offset); err != nil { + t.Fatalf("write %s at %d: %v", name, offset, err) + } +} + +func expectContent(t *testing.T, m *MountedFS, name string, want []byte) { + t.Helper() + got, err := m.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + if bytes.Equal(got, want) { + return + } + i := 0 + for i < len(got) && i < len(want) && got[i] == want[i] { + i++ + } + t.Errorf("%s: %d bytes, want %d; first difference at offset %d", name, len(got), len(want), i) +} + // TestNetworkPartition simulates network partition handling by testing node isolation // and reconnection. Since Go integration tests run multiple FUSE mount processes on // the same host (sharing a backing image), true network partition cannot be simulated. diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index cb54cfb3..c7c2b7f1 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -369,6 +369,25 @@ scenarios: docker: run-tests.sh::test_inode_reuse_stale_read vm: run-vm-tests.sh::test_inode_reuse_stale_read + - id: multinode-extent-cache-coherency + name: Extent cache coherency across nodes + description: One node changes the block mappings of files another node has read (truncate and rewrite, hole fill); the other node reads the new data + priority: critical + environments: + go: multinode_test.go::TestMultiNodeExtentCacheCoherency + docker: run-tests.sh::test_extent_cache_coherency + vm: run-vm-tests.sh::test_extent_cache_coherency + + # The Docker and VM harnesses have no per-node mount flags, so they cannot run one node with -attr-timeout=0. + - id: multinode-attr-timeout-zero + name: Attribute timeout 0 shows remote size changes + description: A node mounted with -attr-timeout=0 sees another node's size change on its next stat + priority: medium + environments: + go: multinode_test.go::TestMultiNodeAttrTimeoutZero + docker: TODO + vm: TODO + # =========================================================================== # NETWORK AND RESILIENCE TESTS # =========================================================================== @@ -1169,6 +1188,34 @@ scenarios: environments: go: internal/cache/inode_cache_test.go::TestCacheInodeClearsNegativeEntry + - id: inode-cache-keeps-newer + name: Inode cache keeps the newer copy + description: Caching an older copy of an inode does not replace a newer cached copy + priority: go_only + environments: + go: internal/cache/inode_cache_test.go::TestCacheInodeKeepsNewerCopy + + - id: extent-cache-inode-version + name: Extent cache misses after an inode write + description: A cached block mapping is not used with a later version of its inode + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestGetExtentMissesAfterInodeWrite + + - id: extent-cache-invalidate + name: Extent invalidation drops the cached inode + description: InvalidateExtentsForInode forces the next lookup to read the current inode version + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestInvalidateExtentsForInodeDropsCachedInode + + - id: extent-cache-stale-miss + name: Stale extent lookups count as misses + description: A cached mapping for an older inode version is removed and counted as a miss, not a hit + priority: go_only + environments: + go: internal/cache/extent_cache_test.go::TestGetExtentCountsStaleEntryAsMiss + - id: broadcast-ignore-own name: Broadcast ignores own messages description: Broadcaster ignores messages from itself diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 4aa2f080..6d64598c 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -1694,6 +1694,51 @@ test_inode_reuse_stale_read() { return 0 } +# Test: Extent cache coherency across VMs +# VM 2 reads files, VM 1 changes their block mappings (truncate and rewrite, +# hole fill), and VM 2 must read the new data. +test_extent_cache_coherency() { + log_info "Testing extent cache coherency across VMs..." + + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping extent cache coherency test (requires 2+ VMs)" + return 0 + fi + + ensure_cluster_healthy 2 + + local writer=1 + local reader=2 + local dir=/mnt/goclusterfs + local trunc="extent_trunc_$(date +%s).bin" + local holes="extent_holes_$(date +%s).bin" + + ssh_run "$writer" "head -c 307200 /dev/zero | tr '\\0' o > $dir/$trunc" + ssh_run "$writer" "head -c 4096 /dev/zero | tr '\\0' a > $dir/$holes && head -c 4096 /dev/zero | tr '\\0' c | dd of=$dir/$holes bs=1M seek=2 conv=notrunc 2>/dev/null" + ssh_run "$reader" "cat $dir/$trunc $dir/$holes > /dev/null" + + # The padding file takes the freed block, so the rewrite lands in another one. + ssh_run "$writer" "truncate -s 0 $dir/$trunc && head -c 307200 /dev/zero | tr '\\0' p > $dir/extent_pad.bin && head -c 307200 /dev/zero | tr '\\0' n > $dir/$trunc" + ssh_run "$writer" "head -c 4096 /dev/zero | tr '\\0' b | dd of=$dir/$holes bs=1M seek=1 conv=notrunc 2>/dev/null" + + local f want got failed=0 + for f in "$trunc" "$holes"; do + want=$(ssh_run "$writer" "md5sum < $dir/$f") + got=$(ssh_run "$reader" "md5sum < $dir/$f") + if [[ "$got" != "$want" ]]; then + log_error "VM $reader read stale data from $f: md5 '$got', want '$want'" + failed=1 + fi + done + ssh_run "$writer" "rm -f $dir/$trunc $dir/$holes $dir/extent_pad.bin" + + if [[ $failed -ne 0 ]]; then + return 1 + fi + log_info "Extent cache coherency test passed" + return 0 +} + # Print the free 1MB blocks that statfs reports on VM $1 statfs_free_blocks_on_vm() { ssh_run "$1" "stat -f -c %f /mnt/goclusterfs" @@ -3236,6 +3281,7 @@ main() { run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional run_test "Inode Reuse Stale Read" test_inode_reuse_stale_read + run_test "Extent Cache Coherency" test_extent_cache_coherency run_test "Statfs After Remount" test_statfs_after_remount run_test "Statfs Multi-Node" test_statfs_multinode run_test "Rolling Restart" test_rolling_restart