From 3381df55763fe7fd3c8ae329ac57d4ec23e94042 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:56:17 -0500 Subject: [PATCH 1/2] fix(refcount): never read or write past the refcount region The refcount region is 1 MB, which holds entries for 130816 blocks (127.75 GiB). Region bounded block numbers only by the volume's block count, so on larger volumes the sectors of higher blocks fell in the snapshot registry and, past about 255.5 GiB, in data blocks. Registry sectors use the same CRC32C format, so a snapshot or clone of a high block, or a plain delete, rewrote registry bytes without an error. NewRegion now takes the region size. Blocks past it read as having a single owner and are never written; sharing them fails with ErrBlockPastRegion. Scan and Initialize stay inside the region, and mount warns when the volume has more blocks than the region covers. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 6 +- internal/extent/extent_test.go | 8 +- internal/fsck/check_fuzz_test.go | 2 +- internal/goclusterfs/fsck_test.go | 2 +- internal/goclusterfs/snapshot_test.go | 2 +- internal/refcount/refcount.go | 50 +++++++++- internal/refcount/refcount_test.go | 130 +++++++++++++++++++------- internal/snapshot/worker_test.go | 2 +- tests/test-manifest.yaml | 7 ++ 9 files changed, 164 insertions(+), 45 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e5e6cf44..cdcc3ba4 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -274,8 +274,12 @@ func main() { } // Refcounts stay on without snapshots: blocks shared earlier still need copy-on-write. - refcountReg := refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) + refcountReg := refcount.NewRegion(dev, refcountOffset, sb.RefCountTableSize, totalBlocks, lockMgr) log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) + if covered := refcount.BlocksForSize(sb.RefCountTableSize); totalBlocks > covered { + log.Printf("[WARN] Refcount: region covers blocks 0-%d of %d; snapshots and clones of files with data in higher blocks fail", + covered-1, totalBlocks) + } // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index 90f7bc3c..393dced9 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -1186,7 +1186,7 @@ func createTestSetupWithRefcount(tb testing.TB) (*Manager, *refcount.Region, *in refcountSize := uint64(1 << 20) // 1MB for refcounts bm := bitmap.NewRegion(dev, bitmapOffset, 1000, nil, uuid) // 1000 blocks - refcountReg := refcount.NewRegion(dev, refcountOffset, refcountSize, nil) + refcountReg := refcount.NewRegion(dev, refcountOffset, refcountSize, 1000, nil) mgr := NewManager(dev, bm, nil, 2<<20) ino := &inode.Inode{ @@ -1511,7 +1511,7 @@ func listExtents(tb testing.TB, mgr *Manager, ino *inode.Inode) []*Extent { func TestFreeKeepsSharedBlocks(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) initial := env.bm.FreeCount() @@ -1574,7 +1574,7 @@ func legacyShared(t *testing.T, env *treeEnv, refs *refcount.Region, count uint3 func TestUncountedSharingIsNeverFreed(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) truncate := func(ino *inode.Inode) { t.Helper() @@ -1616,7 +1616,7 @@ func TestUncountedSharingIsNeverFreed(t *testing.T) { // inode write may land, so the old block is released once. func TestConcurrentInlineCopyOnWrite(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) src := env.newInode() diff --git a/internal/fsck/check_fuzz_test.go b/internal/fsck/check_fuzz_test.go index 75b7ee49..6376efeb 100644 --- a/internal/fsck/check_fuzz_test.go +++ b/internal/fsck/check_fuzz_test.go @@ -82,7 +82,7 @@ func fuzzImage(tb testing.TB) (*storage.Device, []fuzzTarget) { tbl := inode.NewTable(dev, fuzzInodes, 4096*inode.InodeSize, nil, uuid) mgr := extent.NewManager(dev, bm, nil, fuzzDataOffset) tbl.SetTreeFlusher(mgr) - refs := refcount.NewRegion(dev, fuzzRefcounts, fuzzBlocks, nil) + refs := refcount.NewRegion(dev, fuzzRefcounts, refcount.SizeForBlocks(fuzzBlocks), fuzzBlocks, nil) mgr.SetRefcounts(refs) dirs := dir.NewManager(dev, tbl, mgr, nil, fuzzDataOffset) diff --git a/internal/goclusterfs/fsck_test.go b/internal/goclusterfs/fsck_test.go index a16355fc..40f21f69 100644 --- a/internal/goclusterfs/fsck_test.go +++ b/internal/goclusterfs/fsck_test.go @@ -71,7 +71,7 @@ func openFormattedFS(t *testing.T, path string) *FS { inodeTbl := inode.NewTable(dev, sb.InodeTableOffset, sb.InodeCount*inode.InodeSize, locks, nodeUUID) extentMgr := extent.NewManager(dev, bm, locks, sb.DataRegionOffset) inodeTbl.SetTreeFlusher(extentMgr) - refs := refcount.NewRegion(dev, sb.RefCountTableOffset, sb.TotalBlocks, locks) + refs := refcount.NewRegion(dev, sb.RefCountTableOffset, sb.RefCountTableSize, sb.TotalBlocks, locks) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, sb.DataRegionOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, sb.DataRegionOffset) if err := fsys.InitSnapshots(ctx, sb.SnapshotRegOffset, sb.SnapshotRegSize, locks, slot); err != nil { diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 4469e2e8..9e071c0f 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -71,7 +71,7 @@ func openSnapshotFS(t *testing.T, path string) *FS { inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, locks, nodeUUID) extentMgr := extent.NewManager(dev, bm, locks, snapFSDataOffset) inodeTbl.SetTreeFlusher(extentMgr) - refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, locks) + refs := refcount.NewRegion(dev, snapFSRefcountOffset, refcount.SizeForBlocks(snapFSDataBlocks), snapFSDataBlocks, locks) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, snapFSDataOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, snapFSDataOffset) if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, locks, slot); err != nil { diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 186ff0f0..863fbf2b 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -73,6 +73,7 @@ var ( ErrRefCountUnderflow = errors.New("reference count underflow") ErrInvalidChecksum = errors.New("refcount sector checksum mismatch") ErrAlreadyZero = errors.New("reference count already zero") + ErrBlockPastRegion = errors.New("block has no refcount entry: the refcount region is too small for this volume") ) // Entry represents a single reference count entry. @@ -107,6 +108,7 @@ type Region struct { mu sync.RWMutex dev *storage.Device offset uint64 // RefCount region start offset on device + capacity uint64 // Blocks the region has entries for totalBlocks uint64 // Total number of data blocks tracked lockMgr *lock.Manager // For distributed locking (optional) } @@ -116,12 +118,17 @@ type Region struct { // Parameters: // - dev: The storage device // - offset: Byte offset where refcount region starts on device +// - size: Size of the refcount region in bytes; nothing past it is touched // - totalBlocks: Total number of data blocks to track // - lockMgr: Lock manager for distributed locking (can be nil for single-node) -func NewRegion(dev *storage.Device, offset, totalBlocks uint64, lockMgr *lock.Manager) *Region { +// +// Blocks past the region's capacity read as having a single owner, and any +// change to them fails with ErrBlockPastRegion. +func NewRegion(dev *storage.Device, offset, size, totalBlocks uint64, lockMgr *lock.Manager) *Region { return &Region{ dev: dev, offset: offset, + capacity: BlocksForSize(size), totalBlocks: totalBlocks, lockMgr: lockMgr, } @@ -376,6 +383,17 @@ func (r *Region) updateBatch(ctx context.Context, blockNums []uint64, fn func(i sectors := make(map[uint64][]byte) dirty := make(map[uint64]bool) for i, blockNum := range blockNums { + if blockNum >= r.capacity { + entry := Entry{} + changed, err := fn(i, &entry) + if err != nil { + return err + } + if changed && entry != (Entry{}) { + return r.pastRegion(blockNum) + } + continue + } sectorIdx := blockNum / EntriesPerSector if _, ok := sectors[sectorIdx]; !ok { data, err := r.readSector(sectorIdx) @@ -409,7 +427,8 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { r.mu.RLock() defer r.mu.RUnlock() - numSectors := (r.totalBlocks + EntriesPerSector - 1) / EntriesPerSector + covered := r.coveredBlocks() + numSectors := (covered + EntriesPerSector - 1) / EntriesPerSector for sectorIdx := uint64(0); sectorIdx < numSectors; sectorIdx++ { sectorData, err := r.readSector(sectorIdx) @@ -420,7 +439,7 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { baseBlock := sectorIdx * EntriesPerSector for entryIdx := uint64(0); entryIdx < EntriesPerSector; entryIdx++ { blockNum := baseBlock + entryIdx - if blockNum >= r.totalBlocks { + if blockNum >= covered { break } @@ -441,6 +460,7 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { } } + stats.FreeBlocks += r.totalBlocks - covered stats.TotalBlocks = r.totalBlocks return stats, nil } @@ -460,8 +480,19 @@ func (r *Region) TotalBlocks() uint64 { return r.totalBlocks } +func (r *Region) coveredBlocks() uint64 { + return min(r.totalBlocks, r.capacity) +} + +func (r *Region) pastRegion(blockNum uint64) error { + return fmt.Errorf("%w: block %d, region covers blocks 0-%d", ErrBlockPastRegion, blockNum, r.capacity-1) +} + // readEntry reads a single entry from disk. func (r *Region) readEntry(blockNum uint64) (Entry, error) { + if blockNum >= r.capacity { + return Entry{}, nil + } sectorIdx := blockNum / EntriesPerSector entryIdx := blockNum % EntriesPerSector @@ -475,6 +506,12 @@ func (r *Region) readEntry(blockNum uint64) (Entry, error) { // writeEntry writes a single entry to disk. func (r *Region) writeEntry(blockNum uint64, entry Entry) error { + if blockNum >= r.capacity { + if entry == (Entry{}) { + return nil + } + return r.pastRegion(blockNum) + } sectorIdx := blockNum / EntriesPerSector entryIdx := blockNum % EntriesPerSector @@ -581,13 +618,18 @@ func SizeForBlocks(totalBlocks uint64) uint64 { return numSectors * SectorSize } +// BlocksForSize returns how many blocks a refcount region of size bytes has entries for. +func BlocksForSize(size uint64) uint64 { + return size / SectorSize * EntriesPerSector +} + // Initialize initializes the refcount region with all counts set to 0. // This should be called during filesystem format. func (r *Region) Initialize(ctx context.Context) error { r.mu.Lock() defer r.mu.Unlock() - numSectors := (r.totalBlocks + EntriesPerSector - 1) / EntriesPerSector + numSectors := (r.coveredBlocks() + EntriesPerSector - 1) / EntriesPerSector // Create an empty sector (all zeros) emptyData := make([]byte, BytesPerSector) diff --git a/internal/refcount/refcount_test.go b/internal/refcount/refcount_test.go index 88cbbc98..88e6dc22 100644 --- a/internal/refcount/refcount_test.go +++ b/internal/refcount/refcount_test.go @@ -1,12 +1,16 @@ package refcount import ( + "bytes" "context" + "encoding/binary" + "errors" "os" "path/filepath" "testing" "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" ) // createTestDevice creates a test file for O_DIRECT testing. @@ -157,7 +161,7 @@ func TestNewRegion(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) // 10MB defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) if region == nil { t.Fatal("NewRegion returned nil") @@ -172,7 +176,7 @@ func TestGetEntry(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Get entry from uninitialized region - should be zero @@ -193,7 +197,7 @@ func TestGetOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Get(ctx, 100) @@ -211,7 +215,7 @@ func TestIncrement(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Increment from 0 @@ -246,7 +250,7 @@ func TestIncrementOverflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set to max value @@ -266,7 +270,7 @@ func TestIncrementOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Increment(ctx, 100) @@ -279,7 +283,7 @@ func TestDecrement(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial count to 5 @@ -311,7 +315,7 @@ func TestDecrementUnderflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Try to decrement from 0 - should underflow @@ -325,7 +329,7 @@ func TestDecrementOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Decrement(ctx, 100) @@ -338,7 +342,7 @@ func TestSet(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set count and flags @@ -364,7 +368,7 @@ func TestSetOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() err := region.Set(ctx, 100, 1, FlagNone) @@ -377,7 +381,7 @@ func TestSetFlags(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial count @@ -409,7 +413,7 @@ func TestSetFlagsOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() err := region.SetFlags(ctx, 100, FlagSnapshot) @@ -422,7 +426,7 @@ func TestIncrementBatch(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Increment batch @@ -461,7 +465,7 @@ func TestIncrementBatchEmpty(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() counts, err := region.IncrementBatch(ctx, nil) @@ -485,7 +489,7 @@ func TestIncrementBatchOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.IncrementBatch(ctx, []uint64{50, 100}) @@ -498,7 +502,7 @@ func TestIncrementBatchOverflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set one block to max @@ -518,7 +522,7 @@ func TestDecrementBatch(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial counts @@ -552,7 +556,7 @@ func TestDecrementBatchUnderflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set block 0 to 1, leave block 5 at 0 @@ -572,7 +576,7 @@ func TestShareAndRelease(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() blocks := []uint64{3, 700} count := func(block uint64) uint32 { @@ -627,7 +631,7 @@ func TestScan(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() // Set up mixed refcounts: @@ -682,7 +686,7 @@ func TestInitialize(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // First set some values @@ -763,7 +767,7 @@ func TestPersistence(t *testing.T) { } defer func() { _ = dev.Close() }() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set various refcounts and flags @@ -788,7 +792,7 @@ func TestPersistence(t *testing.T) { } defer func() { _ = dev.Close() }() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() tests := []struct { @@ -822,7 +826,7 @@ func TestMultipleSectors(t *testing.T) { // Create region spanning multiple sectors totalBlocks := uint64(EntriesPerSector*3 + 100) // 3 full sectors + partial - region := NewRegion(dev, 0, totalBlocks, nil) + region := NewRegion(dev, 0, SizeForBlocks(totalBlocks), totalBlocks, nil) ctx := context.Background() // Set entries in different sectors @@ -892,7 +896,7 @@ func TestBatchAcrossSectors(t *testing.T) { defer cleanup() totalBlocks := uint64(EntriesPerSector * 3) - region := NewRegion(dev, 0, totalBlocks, nil) + region := NewRegion(dev, 0, SizeForBlocks(totalBlocks), totalBlocks, nil) ctx := context.Background() // Batch increment blocks from different sectors @@ -933,7 +937,7 @@ func TestRegionOffset(t *testing.T) { // Test region at a non-zero offset offset := uint64(50 * SectorSize) // 50 sectors into device - region := NewRegion(dev, offset, 1000, nil) + region := NewRegion(dev, offset, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set some values @@ -952,13 +956,75 @@ func TestRegionOffset(t *testing.T) { } } +// A 200 GiB volume has 204537 blocks, but its 1 MB refcount region holds entries +// for 130816. The sectors of the other blocks would fall in the snapshot registry. +func TestBlocksPastRegionLeaveTheSnapshotRegistryAlone(t *testing.T) { + ctx := context.Background() + dev, cleanup := createTestDevice(t, 200<<30) + defer cleanup() + sb, err := superblock.Format(dev) + if err != nil { + t.Fatal(err) + } + first := BlocksForSize(sb.RefCountTableSize) + last := sb.TotalBlocks - 1 + if last < first { + t.Fatalf("all %d blocks fit the %d-byte refcount region", sb.TotalBlocks, sb.RefCountTableSize) + } + + // The registry's first sector holds a descriptor sector: data and a CRC32C, like a refcount sector. + registry := storage.NewAlignedBuffer(int(sb.SnapshotRegSize)) + for i := range SectorSize - ChecksumSize { + registry.Bytes()[i] = byte(i%251 + 1) + } + binary.LittleEndian.PutUint32(registry.Bytes()[SectorSize-ChecksumSize:], storage.CRC32C(registry.Bytes()[:SectorSize-ChecksumSize])) + if _, err := dev.WriteAt(registry.Bytes(), int64(sb.SnapshotRegOffset)); err != nil { + t.Fatal(err) + } + + region := NewRegion(dev, sb.RefCountTableOffset, sb.RefCountTableSize, sb.TotalBlocks, nil) + + if _, err := region.Increment(ctx, first); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Increment(%d) = %v, want ErrBlockPastRegion", first, err) + } + if err := region.Share(ctx, []uint64{first - 1, last}); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Share(%d, %d) = %v, want ErrBlockPastRegion", first-1, last, err) + } + if err := region.Set(ctx, last, 2, 0); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Set(%d) = %v, want ErrBlockPastRegion", last, err) + } + if err := region.SetFlags(ctx, first, FlagSnapshot); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("SetFlags(%d) = %v, want ErrBlockPastRegion", first, err) + } + if entry, err := region.Get(ctx, first); err != nil || entry != (Entry{}) { + t.Errorf("Get(%d) = %+v, %v, want a single owner", first, entry, err) + } + if free, err := region.Release(ctx, []uint64{first, last}); err != nil || len(free) != 2 { + t.Errorf("Release(%d, %d) = %v, %v, want both free", first, last, free, err) + } + if entry, _ := region.Get(ctx, first-1); entry.RefCount != 0 { + t.Errorf("failed Share left block %d with count %d", first-1, entry.RefCount) + } + if stats, err := region.Scan(ctx); err != nil || stats.FreeBlocks != sb.TotalBlocks { + t.Errorf("Scan = %+v, %v, want all %d blocks free", stats, err, sb.TotalBlocks) + } + + after := storage.NewAlignedBuffer(int(sb.SnapshotRegSize)) + if _, err := dev.ReadAt(after.Bytes(), int64(sb.SnapshotRegOffset)); err != nil { + t.Fatal(err) + } + if !bytes.Equal(after.Bytes(), registry.Bytes()) { + t.Error("refcount updates for blocks past the region changed the snapshot registry") + } +} + // Benchmarks func BenchmarkGet(b *testing.B) { dev, cleanup := createTestDevice(b, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() b.ResetTimer() @@ -973,7 +1039,7 @@ func BenchmarkIncrement(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, uint64(b.N+1), nil) + region := NewRegion(dev, 0, SizeForBlocks(uint64(b.N+1)), uint64(b.N+1), nil) ctx := context.Background() b.ResetTimer() @@ -988,7 +1054,7 @@ func BenchmarkDecrement(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, uint64(b.N+1), nil) + region := NewRegion(dev, 0, SizeForBlocks(uint64(b.N+1)), uint64(b.N+1), nil) ctx := context.Background() // Pre-set all counts to 1 @@ -1010,7 +1076,7 @@ func BenchmarkIncrementBatch(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() // Create batch of 100 blocks @@ -1031,7 +1097,7 @@ func BenchmarkScan(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() // Set some values diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index fbda8138..f4681725 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -87,7 +87,7 @@ func newWorkerEnv(t *testing.T) *workerEnv { t: t, ctx: context.Background(), reg: NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil), - refs: refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil), + refs: refcount.NewRegion(dev, testRefcountStart, refcount.SizeForBlocks(testDataBlocks), testDataBlocks, nil), bm: bitmap.NewRegion(dev, testBitmapStart, testDataBlocks, nil, nodeUUID), tbl: inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID), done: make(chan *JobResult, 64), diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 72b6c95f..ea0f1e49 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -519,6 +519,13 @@ scenarios: environments: go: internal/lock/reclaim_test.go::TestAdoptOwnerHoldsDeadNodesLocksUntilReleased + - id: refcount-blocks-past-region + name: Refcounts never write past their region + description: On a 200 GiB volume, changes to blocks the 1 MB refcount region has no entry for fail with ErrBlockPastRegion, reads and frees treat them as single-owner, and the snapshot registry after the region is unchanged + priority: go_only + environments: + go: internal/refcount/refcount_test.go::TestBlocksPastRegionLeaveTheSnapshotRegistryAlone + - id: registry-slot-journal-region name: Registry records each slot's journal region description: Register stores a distinct journal region in each slot From 17d9a484ddf9103e4ff4f1def95a1bde3f31f77f Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:18:08 -0500 Subject: [PATCH 2/2] fix(snapshot): refuse blocks past the refcount region before marking shareInto marked the source shared before Share refused blocks past the refcount region, so after a refusal every write to the file took the copy-on-write check. Check that the region has entries for all source blocks first, and report the refusal as ENOTSUP like uncounted sharing. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 10 +++++++- internal/goclusterfs/snapshot_test.go | 37 +++++++++++++++++++++++++++ internal/refcount/refcount.go | 11 ++++++++ tests/test-manifest.yaml | 7 +++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 87e5c32d..ac600e75 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3305,6 +3305,14 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, sti if uncounted { return errUncountedSharing } + // Refuse before marking src shared, so its writes stay off the copy-on-write path. + covered, err := f.extentMgr.DataBlocks(src) + if err != nil { + return err + } + if err := f.refcountReg.CheckCovered(covered); err != nil { + return err + } // A snapshot's data inode is marked shared when it is created and never written again. if src.Flags&inode.FlagSnapshotData == 0 { if src, err = f.updateInode(ctx, srcNum, f.markShared); err != nil { @@ -3344,7 +3352,7 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, sti // sharingErrno maps a failure to share a file's blocks to an errno. func sharingErrno(err error) syscall.Errno { - if errors.Is(err, errUncountedSharing) { + if errors.Is(err, errUncountedSharing) || errors.Is(err, refcount.ErrBlockPastRegion) { return syscall.ENOTSUP } return syscall.EIO diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 9e071c0f..4f34a150 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -298,6 +298,43 @@ func TestSnapshotRefusesUncountedSharing(t *testing.T) { checkFile(t, f, pattern(1, BlockSize)) } +// On a volume larger than its refcount region, blocks past the region cannot +// be shared. Refusing must leave the file as it was, off the copy-on-write path. +func TestSnapshotRefusesBlocksPastRefcountRegion(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + noEntries := refcount.NewRegion(fsys.dev, snapFSRefcountOffset, 0, snapFSDataBlocks, fsys.lockMgr) + fsys.refcountReg = noEntries + fsys.extentMgr.SetRefcounts(noEntries) + + f := newRegularFile(t, fsys) + writeAt(t, f, pattern(1, 2*BlockSize), 0) + + if errno := f.Setxattr(ctx, "user.snapshot", []byte("high"), 0); errno != syscall.ENOTSUP { + t.Fatalf("snapshot of blocks past the refcount region: errno %v, want ENOTSUP", errno) + } + if errno := f.Setxattr(ctx, "user.clone", []byte("/high-clone"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of blocks past the refcount region: errno %v, want ENOTSUP", errno) + } + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.InlineExtents[0].Flags&extent.FlagShared != 0 || ino.Flags&inode.FlagHasSnapshot != 0 { + t.Fatalf("refused share marked the file shared: extent flags %#x, inode flags %#x", ino.InlineExtents[0].Flags, ino.Flags) + } + + free := fsys.bitmapMgr.FreeCount() + writeAt(t, f, pattern(2, 4096), 10) + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("write after a refused snapshot allocated %d blocks, want 0", free-got) + } + checkFile(t, f, overwritten(pattern(1, 2*BlockSize), 10, pattern(2, 4096))) +} + // A new mount cleans up a snapshot a crash left half created or half deleted, // and the file keeps its blocks. func TestSnapshotRecoveryAfterCrash(t *testing.T) { diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 863fbf2b..8d5bdf72 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -480,6 +480,17 @@ func (r *Region) TotalBlocks() uint64 { return r.totalBlocks } +// CheckCovered returns an ErrBlockPastRegion error for the first block the +// region has no entry for. +func (r *Region) CheckCovered(blockNums []uint64) error { + for _, blockNum := range blockNums { + if blockNum >= r.capacity { + return r.pastRegion(blockNum) + } + } + return nil +} + func (r *Region) coveredBlocks() uint64 { return min(r.totalBlocks, r.capacity) } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index ea0f1e49..77ac6785 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -526,6 +526,13 @@ scenarios: environments: go: internal/refcount/refcount_test.go::TestBlocksPastRegionLeaveTheSnapshotRegistryAlone + - id: snapshot-refuses-blocks-past-refcount-region + name: Snapshot and clone refuse blocks the refcount region cannot count + description: Snapshot and clone of a file with blocks past the refcount region fail with ENOTSUP and leave the file unmarked, so later writes do not copy blocks + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRefusesBlocksPastRefcountRegion + - id: registry-slot-journal-region name: Registry records each slot's journal region description: Register stores a distinct journal region in each slot