From 39b19c3144e6be97aeb4f6c5f79b531a72f5ac02 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:04:01 -0500 Subject: [PATCH 1/2] fix: stop fallocated blocks and block 0 from returning wrong data - The first write to a fallocated block cleared the Unwritten flag on the whole extent, so its other blocks returned old disk contents. It now splits off only the written block and zero-fills it before a partial write, so the rest of that block reads as zeros too. - LookupPhysical returned 0 for holes, so File.Read treated data in physical block 0 as a hole. Holes are now extent.Hole. Co-Authored-By: Claude Opus 5.5 --- internal/cache/extent_cache.go | 2 +- internal/extent/extent.go | 42 +++++++----- internal/extent/extent_test.go | 82 +++++++++++------------- internal/extent/tree_edit.go | 24 ------- internal/goclusterfs/data_safety_test.go | 80 +++++++++++++++++++++++ internal/goclusterfs/fs.go | 29 +++++++-- tests/test-manifest.yaml | 17 +++++ 7 files changed, 185 insertions(+), 91 deletions(-) create mode 100644 internal/goclusterfs/data_safety_test.go diff --git a/internal/cache/extent_cache.go b/internal/cache/extent_cache.go index 0d39f776..698595ea 100644 --- a/internal/cache/extent_cache.go +++ b/internal/cache/extent_cache.go @@ -160,7 +160,7 @@ func (m *Manager) LookupPhysicalCached(extMgr *extent.Manager, ino *inode.Inode, return 0, err } - // Cache the result (including holes which have phys=0) + // Cache the result (including holes, which map to extent.Hole) m.CacheExtent(ino.InodeNum, logicalBlock, phys, ino.DataGeneration) return phys, nil diff --git a/internal/extent/extent.go b/internal/extent/extent.go index 047f4e48..6ea3fab0 100644 --- a/internal/extent/extent.go +++ b/internal/extent/extent.go @@ -48,6 +48,10 @@ const ( FlagSFB = 1 << 3 // Stored in SFB container (not full block) ) +// Hole is what LookupPhysical returns for a logical block with no data on disk. +// Physical block 0 is a valid data block. +const Hole = math.MaxUint64 + // Errors var ( ErrExtentNotFound = errors.New("extent not found") @@ -125,12 +129,12 @@ func (m *Manager) Lookup(ino *inode.Inode, logicalBlock uint64) (*Extent, uint64 return &ext, logicalBlock - ext.LogicalBlock, nil } -// LookupPhysical returns the physical block number for a logical block. -// Returns 0 if the block is a hole (not allocated). +// LookupPhysical returns the physical block number for a logical block, or Hole +// if the block is not allocated. func (m *Manager) LookupPhysical(ino *inode.Inode, logicalBlock uint64) (uint64, error) { ext, offset, err := m.Lookup(ino, logicalBlock) if err == ErrExtentNotFound { - return 0, nil // Hole - return 0 + return Hole, nil } if err != nil { return 0, err @@ -139,7 +143,7 @@ func (m *Manager) LookupPhysical(ino *inode.Inode, logicalBlock uint64) (uint64, // Unwritten extents return zeros like holes for read operations. // The extent exists (space is allocated) but data hasn't been written yet. if ext.Flags&FlagUnwritten != 0 { - return 0, nil // Lazy-zero: treat as hole for reads + return Hole, nil } return ext.PhysicalBlock + offset, nil @@ -226,30 +230,34 @@ func contiguousRuns(logicalBlock uint64, blocks []uint64, flags uint32) []Extent return runs } -// ClearUnwritten clears the Unwritten flag for an extent and increments WrittenBlocks. -// Called on first write to an unwritten block. -// Returns true if the flag was cleared (first write), false if already written. -func (m *Manager) ClearUnwritten(ino *inode.Inode, ext *Extent) (bool, error) { +// ClearUnwritten marks one block of an unwritten extent as written, splitting +// the extent so its other blocks still read as zeros. Called on first write to +// the block. Returns true if the block was unwritten, false if already written. +func (m *Manager) ClearUnwritten(ctx context.Context, ino *inode.Inode, logicalBlock uint64) (bool, error) { + ext, _, err := m.Lookup(ino, logicalBlock) + if err != nil { + return false, err + } if ext.Flags&FlagUnwritten == 0 { return false, nil } - e, err := m.edit(context.Background(), ino) + e, err := m.edit(ctx, ino) if err != nil { return false, err } - var length uint64 - found, err := e.update(ext.LogicalBlock, func(x *Extent) { - x.Flags &^= FlagUnwritten - length = x.Length - }) - if err != nil || !found { + removed, err := e.removeRange(logicalBlock, logicalBlock+1) + if err != nil { + return false, err + } + block := removed[0] + block.Flags &^= FlagUnwritten + if err := e.insert(block); err != nil { return false, err } if err := e.commit(); err != nil { return false, err } - ext.Flags &^= FlagUnwritten - ino.WrittenBlocks += length + ino.WrittenBlocks++ return true, nil } diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index ef9e40cf..2f1f5040 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -425,13 +425,13 @@ func TestLookupPhysical(t *testing.T) { t.Errorf("Physical = %d, want %d", phys, expected) } - // Lookup hole (not allocated) should return 0 + // Lookup hole (not allocated) should return Hole phys, err = mgr.LookupPhysical(ino, 5) if err != nil { t.Fatalf("LookupPhysical hole failed: %v", err) } - if phys != 0 { - t.Errorf("Physical for hole = %d, want 0", phys) + if phys != Hole { + t.Errorf("Physical for hole = %d, want Hole", phys) } } @@ -928,9 +928,9 @@ func TestAllocateUnwritten_DoesNotIncrementWrittenBlocks(t *testing.T) { } } -// TestClearUnwritten_ClearsFlagAndIncrementsWritten verifies that ClearUnwritten -// properly clears the flag and increments WrittenBlocks. -func TestClearUnwritten_ClearsFlagAndIncrementsWritten(t *testing.T) { +// TestClearUnwritten_ClearsOnlyThatBlock verifies that writing one block of a +// multi-block unwritten extent leaves the other blocks reading as holes. +func TestClearUnwritten_ClearsOnlyThatBlock(t *testing.T) { dev, cleanup := createTestDevice(t, 10*1024*1024) defer cleanup() @@ -939,46 +939,39 @@ func TestClearUnwritten_ClearsFlagAndIncrementsWritten(t *testing.T) { mgr := NewManager(dev, bm, nil, 0) ctx := context.Background() - ino := &inode.Inode{ - Magic: inode.InodeMagic, - InodeNum: 100, - AllocatedBlocks: 0, - WrittenBlocks: 0, - } - - // Allocate unwritten extent - ext, err := mgr.AllocateUnwritten(ctx, ino, 0, 1) + ino := &inode.Inode{Magic: inode.InodeMagic, InodeNum: 100} + ext, err := mgr.AllocateUnwritten(ctx, ino, 0, 3) if err != nil { t.Fatalf("AllocateUnwritten failed: %v", err) } - - // Verify starting state - if ino.WrittenBlocks != 0 { - t.Fatalf("Expected WrittenBlocks=0 before clear, got %d", ino.WrittenBlocks) + if ext.Length != 3 { + t.Fatalf("want one 3-block extent, got length %d", ext.Length) } - // Clear unwritten flag - cleared, err := mgr.ClearUnwritten(ino, ext) - if err != nil { - t.Fatalf("ClearUnwritten failed: %v", err) + cleared, err := mgr.ClearUnwritten(ctx, ino, 1) + if err != nil || !cleared { + t.Fatalf("ClearUnwritten(1) = %v, %v; want true, nil", cleared, err) } - if !cleared { - t.Error("ClearUnwritten should return true for first clear") + for lb, want := range []uint64{Hole, ext.PhysicalBlock + 1, Hole} { + if got, err := mgr.LookupPhysical(ino, uint64(lb)); err != nil || got != want { + t.Errorf("LookupPhysical(%d) = %d, %v; want %d", lb, got, err, want) + } } - - // Verify FlagUnwritten is now cleared - if ext.Flags&FlagUnwritten != 0 { - t.Errorf("Expected FlagUnwritten to be cleared, got flags=%d", ext.Flags) + if ino.WrittenBlocks != 1 || ino.AllocatedBlocks != 3 { + t.Errorf("WrittenBlocks=%d AllocatedBlocks=%d, want 1 and 3", ino.WrittenBlocks, ino.AllocatedBlocks) } - // FlagAllocated should still be set - if ext.Flags&FlagAllocated == 0 { - t.Errorf("Expected FlagAllocated to remain set, got flags=%d", ext.Flags) + for _, lb := range []uint64{0, 2} { + if cleared, err := mgr.ClearUnwritten(ctx, ino, lb); err != nil || !cleared { + t.Fatalf("ClearUnwritten(%d) = %v, %v; want true, nil", lb, cleared, err) + } } - - // Verify WrittenBlocks was incremented - if ino.WrittenBlocks != 1 { - t.Errorf("Expected WrittenBlocks=1 after clear, got %d", ino.WrittenBlocks) + exts, err := mgr.ListExtents(ino) + if err != nil { + t.Fatal(err) + } + if len(exts) != 1 || exts[0].Length != 3 || exts[0].Flags != FlagAllocated { + t.Errorf("written blocks should merge back into one 3-block extent, got %d extents", len(exts)) } } @@ -1014,7 +1007,7 @@ func TestClearUnwritten_NoOpIfAlreadyWritten(t *testing.T) { initialWrittenBlocks := ino.WrittenBlocks // ClearUnwritten should be no-op - cleared, err := mgr.ClearUnwritten(ino, ext) + cleared, err := mgr.ClearUnwritten(ctx, ino, 0) if err != nil { t.Fatalf("ClearUnwritten failed: %v", err) } @@ -1029,9 +1022,9 @@ func TestClearUnwritten_NoOpIfAlreadyWritten(t *testing.T) { } } -// TestLookupPhysical_ReturnsZeroForUnwritten verifies that LookupPhysical -// returns 0 (hole) for unwritten extents. -func TestLookupPhysical_ReturnsZeroForUnwritten(t *testing.T) { +// TestLookupPhysical_ReturnsHoleForUnwritten verifies that LookupPhysical +// returns Hole for unwritten extents. +func TestLookupPhysical_ReturnsHoleForUnwritten(t *testing.T) { dev, cleanup := createTestDevice(t, 10*1024*1024) defer cleanup() @@ -1051,13 +1044,13 @@ func TestLookupPhysical_ReturnsZeroForUnwritten(t *testing.T) { t.Fatalf("AllocateUnwritten failed: %v", err) } - // LookupPhysical should return 0 (treat as hole for reads) + // LookupPhysical should return Hole (treat as hole for reads) physBlock, err := mgr.LookupPhysical(ino, 0) if err != nil { t.Fatalf("LookupPhysical failed: %v", err) } - if physBlock != 0 { - t.Errorf("Expected physBlock=0 for unwritten extent, got %d", physBlock) + if physBlock != Hole { + t.Errorf("Expected Hole for unwritten extent, got %d", physBlock) } // But Lookup should find the extent @@ -1070,7 +1063,7 @@ func TestLookupPhysical_ReturnsZeroForUnwritten(t *testing.T) { } // Now clear unwritten and LookupPhysical should return the real block - cleared, err := mgr.ClearUnwritten(ino, foundExt) + cleared, err := mgr.ClearUnwritten(ctx, ino, 0) if err != nil { t.Fatalf("ClearUnwritten failed: %v", err) } @@ -1082,7 +1075,6 @@ func TestLookupPhysical_ReturnsZeroForUnwritten(t *testing.T) { if err != nil { t.Fatalf("LookupPhysical after clear failed: %v", err) } - // Note: ext.PhysicalBlock could be 0 (valid block number), so we compare directly if physBlock != ext.PhysicalBlock { t.Errorf("Expected physBlock=%d after clearing unwritten, got %d", ext.PhysicalBlock, physBlock) } diff --git a/internal/extent/tree_edit.go b/internal/extent/tree_edit.go index 7958fc80..e4a950ec 100644 --- a/internal/extent/tree_edit.go +++ b/internal/extent/tree_edit.go @@ -292,16 +292,6 @@ func (e *editor) set(i int, exts []Extent) { e.changed = true } -func (e *editor) find(lb uint64) (leaf, idx int, ok bool, err error) { - leaf = route(e.leaves, lb) - exts, err := e.extentsAt(leaf) - if err != nil { - return 0, 0, false, err - } - idx, ok = find(exts, lb) - return leaf, idx, ok, nil -} - type piece struct { leaf int ext Extent @@ -361,20 +351,6 @@ func (e *editor) removeRange(from, to uint64) ([]Extent, error) { return removed, nil } -// update applies fn to the extent containing lb and reports whether one exists. -func (e *editor) update(lb uint64, fn func(*Extent)) (bool, error) { - leaf, idx, ok, err := e.find(lb) - if err != nil || !ok { - return false, err - } - exts, err := e.mutable(leaf) - if err != nil { - return false, err - } - fn(&exts[idx]) - return true, nil -} - func (e *editor) updateAll(fn func(*Extent)) error { for i := range e.leaves { exts, err := e.mutable(i) diff --git a/internal/goclusterfs/data_safety_test.go b/internal/goclusterfs/data_safety_test.go new file mode 100644 index 00000000..6f2a1f38 --- /dev/null +++ b/internal/goclusterfs/data_safety_test.go @@ -0,0 +1,80 @@ +package goclusterfs + +import ( + "context" + "testing" + + "github.com/supporttools/GoClusterFS/internal/storage" +) + +func fillBlock(t *testing.T, fsys *FS, physBlock uint64, seed byte) { + t.Helper() + buf := storage.GetBuffer(BlockSize) + defer storage.PutBuffer(buf) + copy(buf.Bytes(), pattern(seed, BlockSize)) + if _, err := fsys.dev.WriteAt(buf.Bytes(), int64(fsys.dataOffset+physBlock*BlockSize)); err != nil { + t.Fatalf("fill block %d: %v", physBlock, err) + } +} + +// A partial write into one block of a fallocated range must not expose what +// the disk held before, in that block or in the other fallocated blocks. +func TestFallocatedBlocksReadZerosAfterPartialWrite(t *testing.T) { + path := newTreeFSImage(t) + fsys := openTreeFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatalf("InitRoot: %v", err) + } + + f := newRegularFile(t, fsys) + if errno := f.Allocate(ctx, nil, 0, 3*BlockSize, 0); errno != 0 { + t.Fatalf("Allocate: %v", errno) + } + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + for lb := uint64(0); lb < 3; lb++ { + ext, off, err := fsys.extentMgr.Lookup(ino, lb) + if err != nil { + t.Fatalf("lookup block %d: %v", lb, err) + } + fillBlock(t, fsys, ext.PhysicalBlock+off, byte(0xA0+lb)) + } + + data := pattern(7, 4096) + writeAt(t, f, data, BlockSize+8192) + + verify := func(f *File) { + t.Helper() + checkAt(t, f, make([]byte, BlockSize), 0) + checkAt(t, f, make([]byte, 8192), BlockSize) + checkAt(t, f, data, BlockSize+8192) + checkAt(t, f, make([]byte, BlockSize-8192-4096), BlockSize+8192+4096) + checkAt(t, f, make([]byte, BlockSize), 2*BlockSize) + } + verify(f) + verify(&File{fs: openTreeFS(t, path), inodeNum: f.inodeNum}) +} + +// Physical block 0 is an ordinary data block, not a hole. +func TestReadDataInPhysicalBlockZero(t *testing.T) { + fsys := openTreeFS(t, newTreeFSImage(t)) + f := newRegularFile(t, fsys) + data := pattern(3, 4096) + writeAt(t, f, data, 0) + + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + ext, off, err := fsys.extentMgr.Lookup(ino, 0) + if err != nil { + t.Fatal(err) + } + if phys := ext.PhysicalBlock + off; phys != 0 { + t.Fatalf("file got physical block %d; this test needs block 0", phys) + } + checkAt(t, f, data, 0) +} diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 9b45c5f4..a8802d1a 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -323,7 +323,7 @@ func (f *FS) cacheInode(ino *inode.Inode) { } // lookupPhysical finds the physical block for a logical block, using cache. -// Returns 0 for holes (sparse file regions with no allocated blocks). +// Returns extent.Hole for holes (sparse file regions with no data on disk). // This is the preferred method for read paths. func (f *FS) lookupPhysical(ino *inode.Inode, logicalBlock uint64) (uint64, error) { phys, err := f.lookupPhysicalIn(ino, logicalBlock) @@ -1985,8 +1985,7 @@ func (f *File) Read(ctx context.Context, fh fs.FileHandle, dest []byte, off int6 if err != nil { return nil, syscall.EIO } - if physBlock == 0 { - // Hole - return zeros + if physBlock == extent.Hole { toRead := int(BlockSize - blockOffset) if toRead > readSize-bytesRead { toRead = readSize - bytesRead @@ -2070,6 +2069,14 @@ func retryOnTreeConflict(err error) error { return err } +func (f *FS) zeroBlock(physBlock uint64) error { + buf := storage.GetBuffer(BlockSize) + defer storage.PutBuffer(buf) + clear(buf.Bytes()) + _, err := f.dev.WriteAt(buf.Bytes(), int64(f.dataOffset+physBlock*BlockSize)) + return err +} + // writeInternal performs the actual write operation. // Returns errRetryWrite if a concurrent modification is detected and the caller should retry. func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (int, error) { @@ -2203,6 +2210,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i for bytesWritten < len(data) { blockNum := offset / BlockSize blockOffset := offset % BlockSize + zeroFirst := false // Try to find existing extent first (without lock for read-only check) ext, extOffset, err := f.fs.extentMgr.Lookup(ino, blockNum) @@ -2410,7 +2418,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } // Clear the Unwritten flag (if still set) - cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ino, ext) + cleared, clearErr := f.fs.extentMgr.ClearUnwritten(ctx, ino, blockNum) if clearErr != nil { log.Printf("[DEBUG] Write inode %d: clearing Unwritten flag at block %d failed: %v", f.inodeNum, blockNum, clearErr) @@ -2420,6 +2428,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: cleared Unwritten flag for block %d", f.inodeNum, blockNum) } + zeroFirst = cleared } // Calculate physical location @@ -2431,6 +2440,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i if toWrite > len(data)-bytesWritten { toWrite = len(data) - bytesWritten } + zeroFirst = zeroFirst && toWrite < BlockSize // Acquire extent lock on the physical block to prevent lost-update race. // When multiple nodes write to different offsets within the same block, @@ -2446,6 +2456,17 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i extentLockHeld = true } + // The rest of a block first written here would otherwise expose old disk contents. + if zeroFirst { + if err := f.fs.zeroBlock(physBlock); err != nil { + log.Printf("[DEBUG] Write inode %d: zeroing block %d failed: %v", f.inodeNum, physBlock, err) + if extentLockHeld { + f.fs.lockMgr.ReleaseExtent(physBlock) + } + return 0, syscall.EIO + } + } + // Handle unaligned writes by read-modify-write buf := storage.GetBuffer(storage.SectorSize) alignedOffset := diskOffset &^ (storage.SectorSize - 1) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index a6ade5ac..eef707b3 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -826,6 +826,23 @@ scenarios: environments: go: integration_test.go::TestFallocateUnwrittenBlocks + - id: fallocate-partial-write-zeros + name: Fallocated blocks read as zeros after a partial write + description: A partial write into one block of a fallocated range leaves the rest of that block and the other fallocated blocks reading as zeros, not old disk contents + priority: high + environments: + go: internal/goclusterfs/data_safety_test.go::TestFallocatedBlocksReadZerosAfterPartialWrite + docker: TODO - needs old data in the blocks fallocate hands out; reusing a deleted file's blocks races TRIM (on by default, 5s flush), so a shell test would be flaky + vm: TODO - same reason as docker + + # Block 0 belongs to the root directory on a formatted volume, so FUSE cannot reach this case. + - id: read-physical-block-zero + name: Data in physical block 0 is readable + description: File.Read returns data mapped to physical block 0 instead of treating it as a hole + priority: go_only + environments: + go: internal/goclusterfs/data_safety_test.go::TestReadDataInPhysicalBlockZero + - id: multiblock-file-stat name: Multi-block file stat description: Tests st_blocks for files spanning multiple blocks From 3e6944f41c3a07f1f8f928230c848ad9ae1b371c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:10:48 -0500 Subject: [PATCH 2/2] fix: disable SFB allocation and keep legacy SFB files safe SFB could not allocate a slot in any build: the container header buffer is not aligned for O_DIRECT. Had it worked, headers would land 11 MB too low and File.Read would treat the packed container/slot value as a block number. Writes no longer try to allocate slots, so every file uses extents and -sfb has no effect. For SFB inodes written by older builds: - File.Read reads the slot with aligned I/O, or returns EIO if the size does not fit the slot. - Any write or truncate first moves the data to a full block with aligned I/O, and fails instead of mapping a block without the data. - Unlink and migration free the slot after the inode stops using it, with header I/O at the correct offset. A failure leaks the slot. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- CLAUDE.md | 4 +- README.md | 3 +- docs/API-CONFIG.md | 12 +- docs/DEPLOYMENT.md | 4 +- internal/goclusterfs/data_safety_test.go | 154 +++++++++++ internal/goclusterfs/fs.go | 337 ++++++++--------------- tests/test-manifest.yaml | 27 +- 8 files changed, 297 insertions(+), 247 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38332bcf..21f1e612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Inode cache with generation-based coherency checks, and an extent cache. - UDP multicast cache invalidation between nodes, with optional HMAC authentication (`-broadcast-secret`). - Batched TRIM of freed blocks (`-trim`). -- Experimental: SFB containers for files under 64 KB (`-sfb`). Container I/O uses unaligned buffers that `O_DIRECT` rejects, so files fall back to normal extents. +- SFB containers for files under 64 KB, disabled: every file uses normal extents and `-sfb` has no effect. A file that an older build stored in an SFB slot stays readable, moves to a full block on its next write or truncate, and frees its slot when deleted. - Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Copy-on-write does not trigger yet, so a snapshot does not preserve data, and an in-place write to a clone changes the source file. - Experimental: `goclusterfs-fsck` consistency checker with a repair mode. It fails on newly formatted volumes because `-format` does not write a superblock. - Benchmark suite (`tests/benchmark`) and an `internal/metrics` package. @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- After a partial write into a fallocated range, reads of the other fallocated blocks, and of the rest of the written block, returned old disk contents. They now return zeros. - Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge. - `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects. - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. diff --git a/CLAUDE.md b/CLAUDE.md index 86750ec0..3328d106 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ fusermount3 -u mnt # or Ctrl+C the mo | `internal/cache/broadcast` | UDP multicast/unicast cache invalidation, optional HMAC | | `internal/refcount` | Per-block reference counts for copy-on-write | | `internal/snapshot` | Snapshot registry and background refcount worker | -| `internal/sfb` | Small-file containers: 4/8/16/64 KB slots inside a 1 MB block | +| `internal/sfb` | Small-file containers (4/8/16/64 KB slots in a 1 MB block). Disabled: nothing allocates slots, `goclusterfs` only reads and migrates files older builds stored there | | `internal/trim` | Batched `BLKDISCARD` for freed blocks | | `internal/fsck` | The fsck phases | | `internal/metrics` | Atomic counters, not used by `cmd/mount` yet | @@ -91,7 +91,7 @@ Clone paths are absolute from the filesystem root, and clones can only go in the - Every node uses the same 4 MB journal region at 5 MB. `cmd/mount` does not give each node its own journal slot. - Each file holds at most 2 extents (inline in the inode). The overflow extent tree is being built. - The inode count is fixed at 4096. -- SFB containers fail on real devices: `internal/sfb` writes buffers that are not 4 KB aligned, so `storage.WriteAt` rejects them and small files fall back to full 1 MB blocks. +- SFB is disabled: writes never allocate SFB slots, so every file uses full 1 MB blocks. `internal/sfb` container I/O is broken (unaligned header buffers, no data offset) and stays until it is fixed or removed. - No rename, symlinks, or hard links. ## Testing diff --git a/README.md b/README.md index a5d786dc..1e033244 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ A VMFS-like clustered filesystem written in Go. GoClusterFS enables multiple nod - **Network Cache Invalidation**: UDP multicast for near-instant cross-node cache updates - **Copy-on-Write Snapshots**: O(1) file snapshots with refcount-based block sharing - **Thin Provisioning**: Allocate-on-write with TRIM/BLKDISCARD support -- **Small File Optimization**: SFB containers pack files under 64KB efficiently - **8-Phase fsck**: Comprehensive filesystem consistency checker with repair mode ## Requirements @@ -111,7 +110,7 @@ These are the `goclusterfs-mount` flags. Run `./goclusterfs-mount -h` for the fu | Flag | Default | Description | |------|---------|-------------| -| `-sfb` | `true` | Enable Small File Block containers | +| `-sfb` | `true` | Ignored. SFB containers are disabled and every file uses full blocks | | `-snapshots` | `true` | Enable O(1) file snapshots | ## Multi-Node Setup diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 741784a8..8262fcef 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -72,7 +72,7 @@ These flags are passed when starting `goclusterfs`: | Flag | Type | Default | Description | |------|------|---------|-------------| -| `-sfb` | bool | `true` | Enable Small File Block containers | +| `-sfb` | bool | `true` | Ignored. SFB containers are disabled | | `-snapshots` | bool | `true` | Enable O(1) file snapshots | --- @@ -550,15 +550,7 @@ Optimize for consistency: ### Small Files (< 64KB) -Enable SFB optimization: - -```bash -./goclusterfs -device /dev/device -mount /mnt/fs \ - -sfb \ - -journal -``` - -SFB (Small File Block) containers pack multiple small files into single blocks, reducing fragmentation and improving locality. +SFB containers are disabled, so small files use full 1 MB blocks. `-sfb` is accepted and ignored. ### Large File Streaming diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e1c2be48..dd18c8d4 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -368,8 +368,7 @@ Good for general workloads: ./goclusterfs -device /dev/device -mount /mnt/fs \ -journal \ -cache-ttl 5s \ - -trim \ - -sfb + -trim ``` ### High Consistency @@ -405,7 +404,6 @@ Disable advanced features for debugging: -journal=false \ -broadcast=false \ -trim=false \ - -sfb=false \ -snapshots=false \ -debug ``` diff --git a/internal/goclusterfs/data_safety_test.go b/internal/goclusterfs/data_safety_test.go index 6f2a1f38..00d2ae2f 100644 --- a/internal/goclusterfs/data_safety_test.go +++ b/internal/goclusterfs/data_safety_test.go @@ -4,6 +4,12 @@ import ( "context" "testing" + "github.com/hanwen/go-fuse/v2/fuse" + + "github.com/supporttools/GoClusterFS/internal/dir" + "github.com/supporttools/GoClusterFS/internal/extent" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/sfb" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -78,3 +84,151 @@ func TestReadDataInPhysicalBlockZero(t *testing.T) { } checkAt(t, f, data, 0) } + +func editBlock(t *testing.T, fsys *FS, block uint64, edit func([]byte)) { + t.Helper() + buf := storage.GetBuffer(BlockSize) + defer storage.PutBuffer(buf) + off := int64(fsys.dataOffset + block*BlockSize) + if _, err := fsys.dev.ReadAt(buf.Bytes(), off); err != nil { + t.Fatalf("read block %d: %v", block, err) + } + edit(buf.Bytes()) + if _, err := fsys.dev.WriteAt(buf.Bytes(), off); err != nil { + t.Fatalf("write block %d: %v", block, err) + } +} + +// newSFBContainer lays out an empty SFB container the way older builds meant to. +func newSFBContainer(t *testing.T, fsys *FS, tier sfb.Tier) uint64 { + t.Helper() + blocks, err := fsys.bitmapMgr.Alloc(context.Background(), 1) + if err != nil { + t.Fatal(err) + } + h := sfb.Header{Magic: sfb.ContainerMagic, Version: sfb.ContainerVersion, Tier: tier, + SlotCount: tier.SlotCount(), FreeSlots: tier.SlotCount()} + editBlock(t, fsys, blocks[0], func(b []byte) { copy(b, h.Serialize()) }) + return blocks[0] +} + +// newSFBFile creates a linked file whose data lives in an SFB slot. +func newSFBFile(t *testing.T, fsys *FS, container uint64, tier sfb.Tier, slot uint16, data []byte) *File { + t.Helper() + editBlock(t, fsys, container, func(b []byte) { + h, err := sfb.DeserializeHeader(b) + if err != nil { + t.Fatal(err) + } + h.Bitmap[slot/8] |= 1 << (slot % 8) + h.FreeSlots-- + copy(b, h.Serialize()) + copy(b[sfb.SlotOffset(tier, slot):], data) + }) + f := newRegularFile(t, fsys) + if _, err := fsys.updateInode(context.Background(), f.inodeNum, func(ino *inode.Inode) error { + ino.InlineExtents[0] = inode.InlineExtent{ + PhysicalBlock: sfb.PackPhysicalBlock(container, slot), + Length: 1, + Flags: extent.FlagAllocated | extent.FlagSFB | uint32(tier)<<4, + } + ino.ExtentCount = 1 + ino.Size = uint64(len(data)) + ino.LinkCount = 1 + return nil + }); err != nil { + t.Fatal(err) + } + return f +} + +func sfbSlotUsed(t *testing.T, fsys *FS, container uint64, slot uint16) bool { + t.Helper() + var used bool + editBlock(t, fsys, container, func(b []byte) { + h, err := sfb.DeserializeHeader(b) + if err != nil { + t.Fatal(err) + } + used = h.Bitmap[slot/8]&(1<<(slot%8)) != 0 + }) + return used +} + +func TestSFBInodeReadable(t *testing.T) { + fsys := openTreeFS(t, newTreeFSImage(t)) + container := newSFBContainer(t, fsys, sfb.Tier16KB) + data := pattern(5, 10000) + f := newSFBFile(t, fsys, container, sfb.Tier16KB, 1, data) + + checkAt(t, f, data, 0) + checkAt(t, f, data[4000:4100], 4000) +} + +// Growing an SFB file past its slot moves the data into a full block without +// losing any of it, and frees the slot. +func TestSFBMigrationKeepsData(t *testing.T) { + path := newTreeFSImage(t) + fsys := openTreeFS(t, path) + container := newSFBContainer(t, fsys, sfb.Tier16KB) + data := pattern(5, 10000) + f := newSFBFile(t, fsys, container, sfb.Tier16KB, 1, data) + + tail := pattern(9, 4096) + writeAt(t, f, tail, 70000) + + reopened := &File{fs: openTreeFS(t, path), inodeNum: f.inodeNum} + checkAt(t, reopened, data, 0) + checkAt(t, reopened, make([]byte, 70000-len(data)), int64(len(data))) + checkAt(t, reopened, tail, 70000) + ino, err := reopened.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.IsSFBBacked() { + t.Fatal("inode is still SFB-backed after growing past its slot") + } + if sfbSlotUsed(t, fsys, container, 1) { + t.Fatal("SFB slot still allocated after migration") + } +} + +// Unlinking or truncating an SFB file frees its slot. +func TestSFBSlotFreed(t *testing.T) { + fsys := openTreeFS(t, newTreeFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatalf("InitRoot: %v", err) + } + container := newSFBContainer(t, fsys, sfb.Tier4KB) + files := map[string]*File{} + for slot, name := range []string{"unlinked-at-root", "unlinked-in-dir", "truncated"} { + f := newSFBFile(t, fsys, container, sfb.Tier4KB, uint16(slot), pattern(byte(slot), 3000)) + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + if err := fsys.dirMgr.Add(ctx, root, name, uint64(f.inodeNum), dir.DT_REG); err != nil { + t.Fatalf("add %s: %v", name, err) + } + files[name] = f + } + + if errno := fsys.Unlink(ctx, "unlinked-at-root"); errno != 0 { + t.Fatalf("FS.Unlink: %v", errno) + } + if errno := (&Dir{fs: fsys, inodeNum: inode.InodeRoot}).Unlink(ctx, "unlinked-in-dir"); errno != 0 { + t.Fatalf("Dir.Unlink: %v", errno) + } + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_SIZE + if errno := files["truncated"].Setattr(ctx, nil, in, &fuse.AttrOut{}); errno != 0 { + t.Fatalf("truncate: %v", errno) + } + + for slot := uint16(0); slot < 3; slot++ { + if sfbSlotUsed(t, fsys, container, slot) { + t.Errorf("SFB slot %d still allocated", slot) + } + } +} diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a8802d1a..fac3e201 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -7,6 +7,7 @@ package goclusterfs import ( + "bytes" "context" "errors" "fmt" @@ -953,10 +954,14 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno { childIno.Ctime = now if childIno.LinkCount == 0 { + sfbBacked, sfbSlot := childIno.IsSFBBacked(), childIno.InlineExtents[0].PhysicalBlock if err := f.extentMgr.Delete(ctx, childIno); err != nil { log.Printf("[DEBUG] Unlink %q: freeing blocks failed: %v", name, err) } f.inodeTbl.Free(ctx, childIno.InodeNum) + if sfbBacked { + f.freeSFBSlot(ctx, sfbSlot) + } } else if _, err := f.updateInode(ctx, childIno.InodeNum, func(ino *inode.Inode) error { ino.LinkCount-- ino.Ctime = now @@ -1821,10 +1826,14 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall. // Decrement child link count and potentially free childIno.LinkCount-- if childIno.LinkCount == 0 { + sfbBacked, sfbSlot := childIno.IsSFBBacked(), childIno.InlineExtents[0].PhysicalBlock // Free all extents d.fs.extentMgr.Delete(ctx, childIno) // Free inode d.fs.inodeTbl.Free(ctx, uint32(childInodeNum)) + if sfbBacked { + d.fs.freeSFBSlot(ctx, sfbSlot) + } } else if updated, err := d.fs.updateInode(ctx, uint32(childInodeNum), func(ino *inode.Inode) error { ino.LinkCount--; return nil }); err != nil { log.Printf("[DEBUG] removeEntry %q: updating link count failed: %v", name, err) } else { @@ -1972,6 +1981,15 @@ func (f *File) Read(ctx context.Context, fh fs.FileHandle, dest []byte, off int6 readSize = int(ino.Size - offset) } + if ino.IsSFBBacked() { + data, err := f.fs.readSFB(ino, offset, uint64(readSize)) + if err != nil { + log.Printf("[ERROR] Read inode %d: SFB read failed: %v", f.inodeNum, err) + return nil, syscall.EIO + } + return fuse.ReadResultData(data), 0 + } + // Read data block by block data := make([]byte, readSize) bytesRead := 0 @@ -2120,93 +2138,23 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } }() - // === SFB (Sub-File Block) handling for small files === - // Check if this file uses SFB storage or could benefit from it. - newEndOffset := offset + uint64(len(data)) - - // Case 1: File is already SFB-backed + // SFB allocation is disabled: every file uses extents, and a file written by an + // older build moves out of its SFB slot before the write. if ino.IsSFBBacked() { - if newEndOffset > SFBMaxFileSize { - // Write would exceed SFB limit - migrate to regular extent first - if err := f.migrateToExtent(ctx, ino); err != nil { - log.Printf("[DEBUG] Write inode %d: SFB migration failed: %v", f.inodeNum, err) + if err := f.migrateToExtent(ctx, ino); err != nil { + log.Printf("[DEBUG] Write inode %d: SFB migration failed: %v", f.inodeNum, err) + if err == errRetryWrite { return 0, err } - // Re-read inode after migration - ino, err = f.fs.inodeTbl.Read(f.inodeNum) - if err != nil { - return 0, syscall.EIO - } - // Fall through to regular extent write path below - } else { - // Write fits in SFB - use SFB write path - return f.writeSFB(ctx, ino, data, offset) + return 0, syscall.EIO } - } - - // Case 2: New file (no extents) that fits in SFB - if ino.ExtentCount == 0 && newEndOffset <= SFBMaxFileSize && f.fs.sfbMgr != nil { - // Allocate SFB slot for this small file - containerBlock, slotIndex, err := f.fs.sfbMgr.AllocateSlot(ctx, newEndOffset) - if err == nil { - // === WAL: Log intent to allocate SFB slot === - var sfbIntentSeq uint64 - if f.fs.journal != nil { - txn := f.fs.journal.Begin() - tier, _ := sfb.TierForSize(newEndOffset) - if err := txn.LogSFBAlloc(&journal.SFBAllocPayload{ - InodeNum: uint64(f.inodeNum), - ContainerBlock: containerBlock, - SlotIndex: slotIndex, - Tier: journal.SFBTier(tier), - }); err != nil { - f.fs.journal.Abort(txn) - // Free the slot on journal failure - _ = f.fs.sfbMgr.FreeSlot(ctx, containerBlock, slotIndex) - return 0, err - } - // === COMMIT POINT === - var commitErr error - sfbIntentSeq, commitErr = f.fs.journal.Commit(txn) - if commitErr != nil { - _ = f.fs.sfbMgr.FreeSlot(ctx, containerBlock, slotIndex) - return 0, commitErr - } - } - - // Store the SFB reference in the inode's first extent - tier, _ := sfb.TierForSize(newEndOffset) - ino.InlineExtents[0] = inode.InlineExtent{ - LogicalBlock: 0, - PhysicalBlock: sfb.PackPhysicalBlock(containerBlock, slotIndex), - Length: 1, - Flags: extent.FlagAllocated | extent.FlagSFB | (uint32(tier) << 4), - } - ino.ExtentCount = 1 - ino.AllocatedBlocks = 0 // SFB doesn't count as full blocks - - if err := f.fs.inodeTbl.Write(ctx, ino); err != nil { - _ = f.fs.sfbMgr.FreeSlot(ctx, containerBlock, slotIndex) - return 0, retryOnTreeConflict(err) - } - - // Write data using SFB path - written, writeErr := f.writeSFB(ctx, ino, data, offset) - - // === WAL: Mark SFB allocation intent complete === - if f.fs.journal != nil && sfbIntentSeq > 0 { - if err := f.fs.journal.CompleteIntent(sfbIntentSeq); err != nil { - log.Printf("[DEBUG] Write inode %d SFB alloc: CompleteIntent failed: %v", f.inodeNum, err) - } - } - - return written, writeErr + ino, err = f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + return 0, syscall.EIO } - // SFB allocation failed (e.g., no space) - fall through to regular extent path - log.Printf("[DEBUG] Write inode %d: SFB allocation failed, using extent: %v", f.inodeNum, err) } - // Regular extent-based write path for large files + // Regular extent-based write path for bytesWritten < len(data) { blockNum := offset / BlockSize blockOffset := offset % BlockSize @@ -2678,165 +2626,99 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return bytesWritten, nil } -// writeSFB handles writes to files stored in SFB containers. -// SFB (Sub-File Block) stores small files (<64KB) efficiently in shared 1MB blocks. -func (f *File) writeSFB(ctx context.Context, ino *inode.Inode, data []byte, offset uint64) (int, error) { - // Verify the file is SFB-backed - if !ino.IsSFBBacked() { - return 0, errors.New("file is not SFB-backed") - } - - // Get the SFB extent (should be the first inline extent) - if ino.ExtentCount == 0 { - return 0, errors.New("SFB file has no extent") +// sfbSlotOffset returns the disk offset of an SFB-backed inode's data. SFB +// allocation is disabled; this serves only inodes written by older builds. +func (f *FS) sfbSlotOffset(ino *inode.Inode) (int64, error) { + ext := ino.InlineExtents[0] + container, slot := sfb.UnpackPhysicalBlock(ext.PhysicalBlock) + slotSize, _ := sfb.Tier((ext.Flags >> 4) & 0x3).Info() + if ino.Size > uint64(slotSize) { + return 0, fmt.Errorf("SFB inode %d: size %d exceeds its %d-byte slot", ino.InodeNum, ino.Size, slotSize) } + return int64(f.dataOffset + container*BlockSize + SFBHeaderSize + uint64(slot)*uint64(slotSize)), nil +} - ext := &ino.InlineExtents[0] - containerBlock, slotIndex := sfb.UnpackPhysicalBlock(ext.PhysicalBlock) - - // Determine slot size from tier - tier := sfb.Tier((ext.Flags >> 4) & 0x3) // Tier stored in bits 4-5 of flags - slotSize, _ := tier.Info() - if slotSize == 0 { - // Fallback: determine from container if tier not in flags - slotSize = 64 * 1024 // Assume max slot size +// readSFB reads n bytes at off from an SFB-backed inode's slot. Slots are not +// sector-aligned, so it reads the sectors that cover the range. +func (f *FS) readSFB(ino *inode.Inode, off, n uint64) ([]byte, error) { + start, err := f.sfbSlotOffset(ino) + if err != nil || n == 0 { + return nil, err } - - // Check if write would exceed slot size (needs migration to full extent) - newSize := offset + uint64(len(data)) - if newSize > uint64(slotSize) { - log.Printf("[DEBUG] SFB write inode %d: size %d exceeds slot %d, needs migration", - f.inodeNum, newSize, slotSize) - return 0, errors.New("write exceeds SFB slot size - migration needed") + from := start + int64(off) + aligned := from &^ (storage.SectorSize - 1) + length := (from + int64(n) - aligned + storage.SectorSize - 1) &^ (storage.SectorSize - 1) + buf := storage.GetBuffer(int(length)) + defer storage.PutBuffer(buf) + if _, err := f.dev.ReadAt(buf.Bytes()[:length], aligned); err != nil { + return nil, err } + return bytes.Clone(buf.Bytes()[from-aligned : from-aligned+int64(n)]), nil +} - // Calculate the physical offset within the container block - // Container layout: [Header 64B][Slot 0][Slot 1]...[Slot N] - slotOffset := uint64(SFBHeaderSize) + uint64(slotIndex)*uint64(slotSize) - diskOffset := int64(f.fs.dataOffset + containerBlock*BlockSize + slotOffset + offset) - - // Acquire SFB container lock (Level 7 in lock hierarchy) - if f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeSFBContainer, containerBlock, lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] SFB write inode %d: failed to acquire container lock: %v", f.inodeNum, err) - return 0, syscall.EIO +// freeSFBSlot clears a slot's bit in its container header. A failure leaks the +// slot, which is safe because nothing allocates SFB slots any more. +func (f *FS) freeSFBSlot(ctx context.Context, packed uint64) { + container, slot := sfb.UnpackPhysicalBlock(packed) + if f.lockMgr != nil { + if err := f.lockMgr.Acquire(ctx, lock.LockTypeSFBContainer, container, lock.LockModeExclusive); err != nil { + log.Printf("[WARN] Free SFB slot %d:%d: container lock failed: %v", container, slot, err) + return } - defer f.fs.lockMgr.Release(lock.LockTypeSFBContainer, containerBlock) + defer f.lockMgr.Release(lock.LockTypeSFBContainer, container) } - // Write data to slot using aligned I/O - bytesWritten := 0 buf := storage.GetBuffer(storage.SectorSize) defer storage.PutBuffer(buf) - - alignedOffset := diskOffset &^ (storage.SectorSize - 1) - sectorOffset := diskOffset - alignedOffset - - for bytesWritten < len(data) { - // Read existing sector - if _, err := f.fs.dev.ReadAt(buf.Bytes(), alignedOffset); err != nil { - // New sector - zero it - for i := range buf.Bytes() { - buf.Bytes()[i] = 0 - } - } - - // Modify - copyLen := storage.SectorSize - int(sectorOffset) - if copyLen > len(data)-bytesWritten { - copyLen = len(data) - bytesWritten - } - copy(buf.Bytes()[sectorOffset:], data[bytesWritten:bytesWritten+copyLen]) - - // Write back - if _, err := f.fs.dev.WriteAt(buf.Bytes(), alignedOffset); err != nil { - log.Printf("[DEBUG] SFB write inode %d: WriteAt failed: %v", f.inodeNum, err) - return bytesWritten, syscall.EIO - } - - bytesWritten += copyLen - alignedOffset += storage.SectorSize - sectorOffset = 0 + sector := buf.Bytes()[:storage.SectorSize] + off := int64(f.dataOffset + container*BlockSize) + if _, err := f.dev.ReadAt(sector, off); err != nil { + log.Printf("[WARN] Free SFB slot %d:%d: reading container header failed: %v", container, slot, err) + return } - - // Update inode size if extended - var intentSeq uint64 - if newSize > ino.Size { - ino.Size = newSize - ino.Mtime = time.Now().UnixNano() - - // === WAL: Log intent to update inode size === - if f.fs.journal != nil { - txn := f.fs.journal.Begin() - if err := txn.LogInodeUpdate(&journal.InodeUpdatePayload{ - InodeNum: uint64(f.inodeNum), - UpdateMask: journal.InodeUpdateSize | journal.InodeUpdateMtime, - }); err != nil { - f.fs.journal.Abort(txn) - return bytesWritten, err - } - // === COMMIT POINT === - var commitErr error - intentSeq, commitErr = f.fs.journal.Commit(txn) - if commitErr != nil { - return bytesWritten, commitErr - } - } - - // Write inode - if err := f.fs.inodeTbl.Write(ctx, ino); err != nil { - return bytesWritten, retryOnTreeConflict(err) - } + h, err := sfb.DeserializeHeader(sector) + if err != nil || !bytes.Equal(h.Serialize(), sector[:sfb.HeaderSize]) || slot >= h.SlotCount { + log.Printf("[WARN] Free SFB slot %d:%d: no valid container header, leaking the slot", container, slot) + return } - - // === WAL: Mark intent complete === - if f.fs.journal != nil && intentSeq > 0 { - if err := f.fs.journal.CompleteIntent(intentSeq); err != nil { - log.Printf("[DEBUG] writeSFB inode %d: CompleteIntent failed: %v", f.inodeNum, err) - } + bit := byte(1) << (slot % 8) + if h.Bitmap[slot/8]&bit == 0 { + log.Printf("[WARN] Free SFB slot %d:%d: slot already free", container, slot) + return + } + h.Bitmap[slot/8] &^= bit + h.FreeSlots++ + copy(sector, h.Serialize()) + if _, err := f.dev.WriteAt(sector, off); err != nil { + log.Printf("[WARN] Free SFB slot %d:%d: writing container header failed: %v", container, slot, err) } - - return bytesWritten, nil } -// migrateToExtent migrates an SFB-backed file to regular extent allocation. -// Called when a small file grows beyond the SFB slot size (64KB). +// migrateToExtent moves an SFB-backed file's data into a full block and frees +// its slot. It fails rather than map the file to a block missing its data. func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { if !ino.IsSFBBacked() { return nil // Already using extents } - ext := &ino.InlineExtents[0] - oldContainer, oldSlot := sfb.UnpackPhysicalBlock(ext.PhysicalBlock) - - // Determine old slot size - tier := sfb.Tier((ext.Flags >> 4) & 0x3) - oldSlotSize, _ := tier.Info() - if oldSlotSize == 0 { - oldSlotSize = 64 * 1024 + data, err := f.fs.readSFB(ino, 0, ino.Size) + if err != nil { + return fmt.Errorf("reading SFB slot: %w", err) } + oldSlot := ino.InlineExtents[0].PhysicalBlock + oldContainer, oldSlotIndex := sfb.UnpackPhysicalBlock(oldSlot) - // Allocate a new full 1MB block newBlocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) if err != nil { return fmt.Errorf("failed to allocate block for migration: %w", err) } newBlock := newBlocks[0] - // Read data from old SFB slot - slotOffset := uint64(SFBHeaderSize) + uint64(oldSlot)*uint64(oldSlotSize) - oldDiskOffset := int64(f.fs.dataOffset + oldContainer*BlockSize + slotOffset) - - slotData := make([]byte, ino.Size) - if _, err := f.fs.dev.ReadAt(slotData, oldDiskOffset); err != nil { - // Best effort - continue with partial data - log.Printf("[WARN] Migration read incomplete for inode %d: %v", f.inodeNum, err) - } - - // Write data to new block - newDiskOffset := int64(f.fs.dataOffset + newBlock*BlockSize) - if _, err := f.fs.dev.WriteAt(slotData, newDiskOffset); err != nil { - // Release the allocated block on failure + buf := storage.GetBuffer(BlockSize) + copy(buf.Bytes(), data) + _, err = f.fs.dev.WriteAt(buf.Bytes()[:BlockSize], int64(f.fs.dataOffset+newBlock*BlockSize)) + storage.PutBuffer(buf) + if err != nil { _ = f.fs.bitmapMgr.Free(ctx, newBlock) return fmt.Errorf("failed to write migrated data: %w", err) } @@ -2848,7 +2730,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { if err := txn.LogSFBMigrate(&journal.SFBMigratePayload{ InodeNum: uint64(f.inodeNum), OldContainer: oldContainer, - OldSlot: oldSlot, + OldSlot: oldSlotIndex, NewBlock: newBlock, }); err != nil { f.fs.journal.Abort(txn) @@ -2864,14 +2746,9 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { } } - // Update extent to point to new block (clear SFB flag) - ext.PhysicalBlock = newBlock - ext.LogicalBlock = 0 - ext.Length = 1 - ext.Flags = extent.FlagAllocated // Clear SFB flag - - // Update inode + ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: newBlock, Length: 1, Flags: extent.FlagAllocated} ino.AllocatedBlocks = 1 + ino.WrittenBlocks = 1 ino.Mtime = time.Now().UnixNano() if err := f.fs.inodeTbl.Write(ctx, ino); err != nil { @@ -2882,12 +2759,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { return retryOnTreeConflict(err) } - // Free the old SFB slot (if sfbMgr is available) - if f.fs.sfbMgr != nil { - if err := f.fs.sfbMgr.FreeSlot(ctx, oldContainer, oldSlot); err != nil { - log.Printf("[WARN] Failed to free old SFB slot %d:%d: %v", oldContainer, oldSlot, err) - } - } + f.fs.freeSFBSlot(ctx, oldSlot) // === WAL: Mark intent complete === if f.fs.journal != nil && intentSeq > 0 { @@ -2897,7 +2769,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { } log.Printf("[INFO] Migrated inode %d from SFB (%d:%d) to extent (block %d)", - f.inodeNum, oldContainer, oldSlot, newBlock) + f.inodeNum, oldContainer, oldSlotIndex, newBlock) return nil } @@ -2927,6 +2799,19 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn return syscall.EIO, nil } + if _, resizing := in.GetSize(); resizing && ino.IsSFBBacked() { + if err := f.migrateToExtent(ctx, ino); err != nil { + log.Printf("[DEBUG] Setattr inode %d: SFB migration failed: %v", f.inodeNum, err) + if err == errRetryWrite { + return 0, err + } + return syscall.EIO, nil + } + if ino, err = f.fs.inodeTbl.Read(f.inodeNum); err != nil { + return syscall.EIO, nil + } + } + now := time.Now().UnixNano() // Track what's changing for WAL diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index eef707b3..e4a4fc47 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -853,9 +853,9 @@ scenarios: # =========================================================================== # SFB (SUB-FILE BLOCK) CONTAINERS # =========================================================================== - # SFB provides efficient storage for small files by packing them into shared - # 1MB blocks. Tiers: 4KB (256 slots), 8KB (128), 16KB (64), 64KB (16). - # Files >64KB migrate to full extent allocation. + # SFB allocation is disabled, so the sfb_test.go tests now cover small files on + # full blocks. The sfb-legacy tests use hand-made SFB inodes, which FUSE cannot + # create, to check that files older builds stored in slots stay safe. - id: sfb-small-file-allocation name: SFB small file allocation @@ -920,6 +920,27 @@ scenarios: docker: TODO vm: TODO + - id: sfb-legacy-read + name: Read a legacy SFB inode + description: File.Read returns the data in an SFB slot instead of reading the packed container/slot value as a block number + priority: go_only + environments: + go: internal/goclusterfs/data_safety_test.go::TestSFBInodeReadable + + - id: sfb-legacy-migration + name: Legacy SFB inode migrates without data loss + description: Growing an SFB file past its slot copies its data to a full block with aligned I/O and frees the slot + priority: go_only + environments: + go: internal/goclusterfs/data_safety_test.go::TestSFBMigrationKeepsData + + - id: sfb-legacy-slot-free + name: Legacy SFB slot freed on unlink and truncate + description: Unlink through FS and Dir, and truncate, clear the file's slot in its container header + priority: go_only + environments: + go: internal/goclusterfs/data_safety_test.go::TestSFBSlotFreed + # =========================================================================== # CACHE BROADCAST TESTS (Phase 4: Cache Coherency) # ===========================================================================