diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e5e6cf44..cdcc3ba4 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -274,8 +274,12 @@ func main() { } // Refcounts stay on without snapshots: blocks shared earlier still need copy-on-write. - refcountReg := refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) + refcountReg := refcount.NewRegion(dev, refcountOffset, sb.RefCountTableSize, totalBlocks, lockMgr) log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) + if covered := refcount.BlocksForSize(sb.RefCountTableSize); totalBlocks > covered { + log.Printf("[WARN] Refcount: region covers blocks 0-%d of %d; snapshots and clones of files with data in higher blocks fail", + covered-1, totalBlocks) + } // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index 90f7bc3c..393dced9 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -1186,7 +1186,7 @@ func createTestSetupWithRefcount(tb testing.TB) (*Manager, *refcount.Region, *in refcountSize := uint64(1 << 20) // 1MB for refcounts bm := bitmap.NewRegion(dev, bitmapOffset, 1000, nil, uuid) // 1000 blocks - refcountReg := refcount.NewRegion(dev, refcountOffset, refcountSize, nil) + refcountReg := refcount.NewRegion(dev, refcountOffset, refcountSize, 1000, nil) mgr := NewManager(dev, bm, nil, 2<<20) ino := &inode.Inode{ @@ -1511,7 +1511,7 @@ func listExtents(tb testing.TB, mgr *Manager, ino *inode.Inode) []*Extent { func TestFreeKeepsSharedBlocks(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) initial := env.bm.FreeCount() @@ -1574,7 +1574,7 @@ func legacyShared(t *testing.T, env *treeEnv, refs *refcount.Region, count uint3 func TestUncountedSharingIsNeverFreed(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) truncate := func(ino *inode.Inode) { t.Helper() @@ -1616,7 +1616,7 @@ func TestUncountedSharingIsNeverFreed(t *testing.T) { // inode write may land, so the old block is released once. func TestConcurrentInlineCopyOnWrite(t *testing.T) { env := newTreeEnv(t, 0) - refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, refcount.SizeForBlocks(treeTestBlocks), treeTestBlocks, nil) env.mgr.SetRefcounts(refs) src := env.newInode() diff --git a/internal/fsck/check_fuzz_test.go b/internal/fsck/check_fuzz_test.go index 75b7ee49..6376efeb 100644 --- a/internal/fsck/check_fuzz_test.go +++ b/internal/fsck/check_fuzz_test.go @@ -82,7 +82,7 @@ func fuzzImage(tb testing.TB) (*storage.Device, []fuzzTarget) { tbl := inode.NewTable(dev, fuzzInodes, 4096*inode.InodeSize, nil, uuid) mgr := extent.NewManager(dev, bm, nil, fuzzDataOffset) tbl.SetTreeFlusher(mgr) - refs := refcount.NewRegion(dev, fuzzRefcounts, fuzzBlocks, nil) + refs := refcount.NewRegion(dev, fuzzRefcounts, refcount.SizeForBlocks(fuzzBlocks), fuzzBlocks, nil) mgr.SetRefcounts(refs) dirs := dir.NewManager(dev, tbl, mgr, nil, fuzzDataOffset) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 87e5c32d..ac600e75 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3305,6 +3305,14 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, sti if uncounted { return errUncountedSharing } + // Refuse before marking src shared, so its writes stay off the copy-on-write path. + covered, err := f.extentMgr.DataBlocks(src) + if err != nil { + return err + } + if err := f.refcountReg.CheckCovered(covered); err != nil { + return err + } // A snapshot's data inode is marked shared when it is created and never written again. if src.Flags&inode.FlagSnapshotData == 0 { if src, err = f.updateInode(ctx, srcNum, f.markShared); err != nil { @@ -3344,7 +3352,7 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, sti // sharingErrno maps a failure to share a file's blocks to an errno. func sharingErrno(err error) syscall.Errno { - if errors.Is(err, errUncountedSharing) { + if errors.Is(err, errUncountedSharing) || errors.Is(err, refcount.ErrBlockPastRegion) { return syscall.ENOTSUP } return syscall.EIO diff --git a/internal/goclusterfs/fsck_test.go b/internal/goclusterfs/fsck_test.go index a16355fc..40f21f69 100644 --- a/internal/goclusterfs/fsck_test.go +++ b/internal/goclusterfs/fsck_test.go @@ -71,7 +71,7 @@ func openFormattedFS(t *testing.T, path string) *FS { inodeTbl := inode.NewTable(dev, sb.InodeTableOffset, sb.InodeCount*inode.InodeSize, locks, nodeUUID) extentMgr := extent.NewManager(dev, bm, locks, sb.DataRegionOffset) inodeTbl.SetTreeFlusher(extentMgr) - refs := refcount.NewRegion(dev, sb.RefCountTableOffset, sb.TotalBlocks, locks) + refs := refcount.NewRegion(dev, sb.RefCountTableOffset, sb.RefCountTableSize, sb.TotalBlocks, locks) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, sb.DataRegionOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, sb.DataRegionOffset) if err := fsys.InitSnapshots(ctx, sb.SnapshotRegOffset, sb.SnapshotRegSize, locks, slot); err != nil { diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 4469e2e8..4f34a150 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -71,7 +71,7 @@ func openSnapshotFS(t *testing.T, path string) *FS { inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, locks, nodeUUID) extentMgr := extent.NewManager(dev, bm, locks, snapFSDataOffset) inodeTbl.SetTreeFlusher(extentMgr) - refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, locks) + refs := refcount.NewRegion(dev, snapFSRefcountOffset, refcount.SizeForBlocks(snapFSDataBlocks), snapFSDataBlocks, locks) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, snapFSDataOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, snapFSDataOffset) if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, locks, slot); err != nil { @@ -298,6 +298,43 @@ func TestSnapshotRefusesUncountedSharing(t *testing.T) { checkFile(t, f, pattern(1, BlockSize)) } +// On a volume larger than its refcount region, blocks past the region cannot +// be shared. Refusing must leave the file as it was, off the copy-on-write path. +func TestSnapshotRefusesBlocksPastRefcountRegion(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + noEntries := refcount.NewRegion(fsys.dev, snapFSRefcountOffset, 0, snapFSDataBlocks, fsys.lockMgr) + fsys.refcountReg = noEntries + fsys.extentMgr.SetRefcounts(noEntries) + + f := newRegularFile(t, fsys) + writeAt(t, f, pattern(1, 2*BlockSize), 0) + + if errno := f.Setxattr(ctx, "user.snapshot", []byte("high"), 0); errno != syscall.ENOTSUP { + t.Fatalf("snapshot of blocks past the refcount region: errno %v, want ENOTSUP", errno) + } + if errno := f.Setxattr(ctx, "user.clone", []byte("/high-clone"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of blocks past the refcount region: errno %v, want ENOTSUP", errno) + } + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.InlineExtents[0].Flags&extent.FlagShared != 0 || ino.Flags&inode.FlagHasSnapshot != 0 { + t.Fatalf("refused share marked the file shared: extent flags %#x, inode flags %#x", ino.InlineExtents[0].Flags, ino.Flags) + } + + free := fsys.bitmapMgr.FreeCount() + writeAt(t, f, pattern(2, 4096), 10) + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("write after a refused snapshot allocated %d blocks, want 0", free-got) + } + checkFile(t, f, overwritten(pattern(1, 2*BlockSize), 10, pattern(2, 4096))) +} + // A new mount cleans up a snapshot a crash left half created or half deleted, // and the file keeps its blocks. func TestSnapshotRecoveryAfterCrash(t *testing.T) { diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 186ff0f0..8d5bdf72 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -73,6 +73,7 @@ var ( ErrRefCountUnderflow = errors.New("reference count underflow") ErrInvalidChecksum = errors.New("refcount sector checksum mismatch") ErrAlreadyZero = errors.New("reference count already zero") + ErrBlockPastRegion = errors.New("block has no refcount entry: the refcount region is too small for this volume") ) // Entry represents a single reference count entry. @@ -107,6 +108,7 @@ type Region struct { mu sync.RWMutex dev *storage.Device offset uint64 // RefCount region start offset on device + capacity uint64 // Blocks the region has entries for totalBlocks uint64 // Total number of data blocks tracked lockMgr *lock.Manager // For distributed locking (optional) } @@ -116,12 +118,17 @@ type Region struct { // Parameters: // - dev: The storage device // - offset: Byte offset where refcount region starts on device +// - size: Size of the refcount region in bytes; nothing past it is touched // - totalBlocks: Total number of data blocks to track // - lockMgr: Lock manager for distributed locking (can be nil for single-node) -func NewRegion(dev *storage.Device, offset, totalBlocks uint64, lockMgr *lock.Manager) *Region { +// +// Blocks past the region's capacity read as having a single owner, and any +// change to them fails with ErrBlockPastRegion. +func NewRegion(dev *storage.Device, offset, size, totalBlocks uint64, lockMgr *lock.Manager) *Region { return &Region{ dev: dev, offset: offset, + capacity: BlocksForSize(size), totalBlocks: totalBlocks, lockMgr: lockMgr, } @@ -376,6 +383,17 @@ func (r *Region) updateBatch(ctx context.Context, blockNums []uint64, fn func(i sectors := make(map[uint64][]byte) dirty := make(map[uint64]bool) for i, blockNum := range blockNums { + if blockNum >= r.capacity { + entry := Entry{} + changed, err := fn(i, &entry) + if err != nil { + return err + } + if changed && entry != (Entry{}) { + return r.pastRegion(blockNum) + } + continue + } sectorIdx := blockNum / EntriesPerSector if _, ok := sectors[sectorIdx]; !ok { data, err := r.readSector(sectorIdx) @@ -409,7 +427,8 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { r.mu.RLock() defer r.mu.RUnlock() - numSectors := (r.totalBlocks + EntriesPerSector - 1) / EntriesPerSector + covered := r.coveredBlocks() + numSectors := (covered + EntriesPerSector - 1) / EntriesPerSector for sectorIdx := uint64(0); sectorIdx < numSectors; sectorIdx++ { sectorData, err := r.readSector(sectorIdx) @@ -420,7 +439,7 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { baseBlock := sectorIdx * EntriesPerSector for entryIdx := uint64(0); entryIdx < EntriesPerSector; entryIdx++ { blockNum := baseBlock + entryIdx - if blockNum >= r.totalBlocks { + if blockNum >= covered { break } @@ -441,6 +460,7 @@ func (r *Region) Scan(ctx context.Context) (stats ScanStats, err error) { } } + stats.FreeBlocks += r.totalBlocks - covered stats.TotalBlocks = r.totalBlocks return stats, nil } @@ -460,8 +480,30 @@ func (r *Region) TotalBlocks() uint64 { return r.totalBlocks } +// CheckCovered returns an ErrBlockPastRegion error for the first block the +// region has no entry for. +func (r *Region) CheckCovered(blockNums []uint64) error { + for _, blockNum := range blockNums { + if blockNum >= r.capacity { + return r.pastRegion(blockNum) + } + } + return nil +} + +func (r *Region) coveredBlocks() uint64 { + return min(r.totalBlocks, r.capacity) +} + +func (r *Region) pastRegion(blockNum uint64) error { + return fmt.Errorf("%w: block %d, region covers blocks 0-%d", ErrBlockPastRegion, blockNum, r.capacity-1) +} + // readEntry reads a single entry from disk. func (r *Region) readEntry(blockNum uint64) (Entry, error) { + if blockNum >= r.capacity { + return Entry{}, nil + } sectorIdx := blockNum / EntriesPerSector entryIdx := blockNum % EntriesPerSector @@ -475,6 +517,12 @@ func (r *Region) readEntry(blockNum uint64) (Entry, error) { // writeEntry writes a single entry to disk. func (r *Region) writeEntry(blockNum uint64, entry Entry) error { + if blockNum >= r.capacity { + if entry == (Entry{}) { + return nil + } + return r.pastRegion(blockNum) + } sectorIdx := blockNum / EntriesPerSector entryIdx := blockNum % EntriesPerSector @@ -581,13 +629,18 @@ func SizeForBlocks(totalBlocks uint64) uint64 { return numSectors * SectorSize } +// BlocksForSize returns how many blocks a refcount region of size bytes has entries for. +func BlocksForSize(size uint64) uint64 { + return size / SectorSize * EntriesPerSector +} + // Initialize initializes the refcount region with all counts set to 0. // This should be called during filesystem format. func (r *Region) Initialize(ctx context.Context) error { r.mu.Lock() defer r.mu.Unlock() - numSectors := (r.totalBlocks + EntriesPerSector - 1) / EntriesPerSector + numSectors := (r.coveredBlocks() + EntriesPerSector - 1) / EntriesPerSector // Create an empty sector (all zeros) emptyData := make([]byte, BytesPerSector) diff --git a/internal/refcount/refcount_test.go b/internal/refcount/refcount_test.go index 88cbbc98..88e6dc22 100644 --- a/internal/refcount/refcount_test.go +++ b/internal/refcount/refcount_test.go @@ -1,12 +1,16 @@ package refcount import ( + "bytes" "context" + "encoding/binary" + "errors" "os" "path/filepath" "testing" "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" ) // createTestDevice creates a test file for O_DIRECT testing. @@ -157,7 +161,7 @@ func TestNewRegion(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) // 10MB defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) if region == nil { t.Fatal("NewRegion returned nil") @@ -172,7 +176,7 @@ func TestGetEntry(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Get entry from uninitialized region - should be zero @@ -193,7 +197,7 @@ func TestGetOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Get(ctx, 100) @@ -211,7 +215,7 @@ func TestIncrement(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Increment from 0 @@ -246,7 +250,7 @@ func TestIncrementOverflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set to max value @@ -266,7 +270,7 @@ func TestIncrementOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Increment(ctx, 100) @@ -279,7 +283,7 @@ func TestDecrement(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial count to 5 @@ -311,7 +315,7 @@ func TestDecrementUnderflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Try to decrement from 0 - should underflow @@ -325,7 +329,7 @@ func TestDecrementOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.Decrement(ctx, 100) @@ -338,7 +342,7 @@ func TestSet(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set count and flags @@ -364,7 +368,7 @@ func TestSetOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() err := region.Set(ctx, 100, 1, FlagNone) @@ -377,7 +381,7 @@ func TestSetFlags(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial count @@ -409,7 +413,7 @@ func TestSetFlagsOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() err := region.SetFlags(ctx, 100, FlagSnapshot) @@ -422,7 +426,7 @@ func TestIncrementBatch(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Increment batch @@ -461,7 +465,7 @@ func TestIncrementBatchEmpty(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() counts, err := region.IncrementBatch(ctx, nil) @@ -485,7 +489,7 @@ func TestIncrementBatchOutOfRange(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() _, err := region.IncrementBatch(ctx, []uint64{50, 100}) @@ -498,7 +502,7 @@ func TestIncrementBatchOverflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set one block to max @@ -518,7 +522,7 @@ func TestDecrementBatch(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set initial counts @@ -552,7 +556,7 @@ func TestDecrementBatchUnderflow(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set block 0 to 1, leave block 5 at 0 @@ -572,7 +576,7 @@ func TestShareAndRelease(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() blocks := []uint64{3, 700} count := func(block uint64) uint32 { @@ -627,7 +631,7 @@ func TestScan(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 100, nil) + region := NewRegion(dev, 0, SizeForBlocks(100), 100, nil) ctx := context.Background() // Set up mixed refcounts: @@ -682,7 +686,7 @@ func TestInitialize(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // First set some values @@ -763,7 +767,7 @@ func TestPersistence(t *testing.T) { } defer func() { _ = dev.Close() }() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set various refcounts and flags @@ -788,7 +792,7 @@ func TestPersistence(t *testing.T) { } defer func() { _ = dev.Close() }() - region := NewRegion(dev, 0, 1000, nil) + region := NewRegion(dev, 0, SizeForBlocks(1000), 1000, nil) ctx := context.Background() tests := []struct { @@ -822,7 +826,7 @@ func TestMultipleSectors(t *testing.T) { // Create region spanning multiple sectors totalBlocks := uint64(EntriesPerSector*3 + 100) // 3 full sectors + partial - region := NewRegion(dev, 0, totalBlocks, nil) + region := NewRegion(dev, 0, SizeForBlocks(totalBlocks), totalBlocks, nil) ctx := context.Background() // Set entries in different sectors @@ -892,7 +896,7 @@ func TestBatchAcrossSectors(t *testing.T) { defer cleanup() totalBlocks := uint64(EntriesPerSector * 3) - region := NewRegion(dev, 0, totalBlocks, nil) + region := NewRegion(dev, 0, SizeForBlocks(totalBlocks), totalBlocks, nil) ctx := context.Background() // Batch increment blocks from different sectors @@ -933,7 +937,7 @@ func TestRegionOffset(t *testing.T) { // Test region at a non-zero offset offset := uint64(50 * SectorSize) // 50 sectors into device - region := NewRegion(dev, offset, 1000, nil) + region := NewRegion(dev, offset, SizeForBlocks(1000), 1000, nil) ctx := context.Background() // Set some values @@ -952,13 +956,75 @@ func TestRegionOffset(t *testing.T) { } } +// A 200 GiB volume has 204537 blocks, but its 1 MB refcount region holds entries +// for 130816. The sectors of the other blocks would fall in the snapshot registry. +func TestBlocksPastRegionLeaveTheSnapshotRegistryAlone(t *testing.T) { + ctx := context.Background() + dev, cleanup := createTestDevice(t, 200<<30) + defer cleanup() + sb, err := superblock.Format(dev) + if err != nil { + t.Fatal(err) + } + first := BlocksForSize(sb.RefCountTableSize) + last := sb.TotalBlocks - 1 + if last < first { + t.Fatalf("all %d blocks fit the %d-byte refcount region", sb.TotalBlocks, sb.RefCountTableSize) + } + + // The registry's first sector holds a descriptor sector: data and a CRC32C, like a refcount sector. + registry := storage.NewAlignedBuffer(int(sb.SnapshotRegSize)) + for i := range SectorSize - ChecksumSize { + registry.Bytes()[i] = byte(i%251 + 1) + } + binary.LittleEndian.PutUint32(registry.Bytes()[SectorSize-ChecksumSize:], storage.CRC32C(registry.Bytes()[:SectorSize-ChecksumSize])) + if _, err := dev.WriteAt(registry.Bytes(), int64(sb.SnapshotRegOffset)); err != nil { + t.Fatal(err) + } + + region := NewRegion(dev, sb.RefCountTableOffset, sb.RefCountTableSize, sb.TotalBlocks, nil) + + if _, err := region.Increment(ctx, first); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Increment(%d) = %v, want ErrBlockPastRegion", first, err) + } + if err := region.Share(ctx, []uint64{first - 1, last}); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Share(%d, %d) = %v, want ErrBlockPastRegion", first-1, last, err) + } + if err := region.Set(ctx, last, 2, 0); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("Set(%d) = %v, want ErrBlockPastRegion", last, err) + } + if err := region.SetFlags(ctx, first, FlagSnapshot); !errors.Is(err, ErrBlockPastRegion) { + t.Errorf("SetFlags(%d) = %v, want ErrBlockPastRegion", first, err) + } + if entry, err := region.Get(ctx, first); err != nil || entry != (Entry{}) { + t.Errorf("Get(%d) = %+v, %v, want a single owner", first, entry, err) + } + if free, err := region.Release(ctx, []uint64{first, last}); err != nil || len(free) != 2 { + t.Errorf("Release(%d, %d) = %v, %v, want both free", first, last, free, err) + } + if entry, _ := region.Get(ctx, first-1); entry.RefCount != 0 { + t.Errorf("failed Share left block %d with count %d", first-1, entry.RefCount) + } + if stats, err := region.Scan(ctx); err != nil || stats.FreeBlocks != sb.TotalBlocks { + t.Errorf("Scan = %+v, %v, want all %d blocks free", stats, err, sb.TotalBlocks) + } + + after := storage.NewAlignedBuffer(int(sb.SnapshotRegSize)) + if _, err := dev.ReadAt(after.Bytes(), int64(sb.SnapshotRegOffset)); err != nil { + t.Fatal(err) + } + if !bytes.Equal(after.Bytes(), registry.Bytes()) { + t.Error("refcount updates for blocks past the region changed the snapshot registry") + } +} + // Benchmarks func BenchmarkGet(b *testing.B) { dev, cleanup := createTestDevice(b, 10<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() b.ResetTimer() @@ -973,7 +1039,7 @@ func BenchmarkIncrement(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, uint64(b.N+1), nil) + region := NewRegion(dev, 0, SizeForBlocks(uint64(b.N+1)), uint64(b.N+1), nil) ctx := context.Background() b.ResetTimer() @@ -988,7 +1054,7 @@ func BenchmarkDecrement(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, uint64(b.N+1), nil) + region := NewRegion(dev, 0, SizeForBlocks(uint64(b.N+1)), uint64(b.N+1), nil) ctx := context.Background() // Pre-set all counts to 1 @@ -1010,7 +1076,7 @@ func BenchmarkIncrementBatch(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() // Create batch of 100 blocks @@ -1031,7 +1097,7 @@ func BenchmarkScan(b *testing.B) { dev, cleanup := createTestDevice(b, 100<<20) defer cleanup() - region := NewRegion(dev, 0, 10000, nil) + region := NewRegion(dev, 0, SizeForBlocks(10000), 10000, nil) ctx := context.Background() // Set some values diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index fbda8138..f4681725 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -87,7 +87,7 @@ func newWorkerEnv(t *testing.T) *workerEnv { t: t, ctx: context.Background(), reg: NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil), - refs: refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil), + refs: refcount.NewRegion(dev, testRefcountStart, refcount.SizeForBlocks(testDataBlocks), testDataBlocks, nil), bm: bitmap.NewRegion(dev, testBitmapStart, testDataBlocks, nil, nodeUUID), tbl: inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID), done: make(chan *JobResult, 64), diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 72b6c95f..77ac6785 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -519,6 +519,20 @@ scenarios: environments: go: internal/lock/reclaim_test.go::TestAdoptOwnerHoldsDeadNodesLocksUntilReleased + - id: refcount-blocks-past-region + name: Refcounts never write past their region + description: On a 200 GiB volume, changes to blocks the 1 MB refcount region has no entry for fail with ErrBlockPastRegion, reads and frees treat them as single-owner, and the snapshot registry after the region is unchanged + priority: go_only + environments: + go: internal/refcount/refcount_test.go::TestBlocksPastRegionLeaveTheSnapshotRegistryAlone + + - id: snapshot-refuses-blocks-past-refcount-region + name: Snapshot and clone refuse blocks the refcount region cannot count + description: Snapshot and clone of a file with blocks past the refcount region fail with ENOTSUP and leave the file unmarked, so later writes do not copy blocks + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRefusesBlocksPastRefcountRegion + - id: registry-slot-journal-region name: Registry records each slot's journal region description: Register stores a distinct journal region in each slot