diff --git a/CHANGELOG.md b/CHANGELOG.md index 960f95f9..225d4e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. +- Run `goclusterfs-fsck` before the first mount after upgrading an image that has snapshots or clones. Phase 5 reports blocks shared by more files than their refcount allows. +- Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. This version never frees such blocks, so deleting those files leaks their space. It also refuses to snapshot or clone them (`ENOTSUP`), and in-place writes to an old clone still change its source. To get files this version can snapshot and free, copy each affected file with `cp` and delete the original. +- Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. ### Added @@ -26,7 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - UDP multicast cache invalidation between nodes, with optional HMAC authentication (`-broadcast-secret`). - Batched TRIM of freed blocks (`-trim`). - SFB containers for files under 64 KB, disabled: every file uses normal extents and `-sfb` has no effect. A file that an older build stored in an SFB slot stays readable, moves to a full block on its next write or truncate, and frees its slot when deleted. -- Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Copy-on-write does not trigger yet, so a snapshot does not preserve data, and an in-place write to a clone changes the source file. +- Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Read a snapshot by cloning it with `user.snapshot.clone`. - Experimental: `goclusterfs-fsck` consistency checker with a repair mode. It reports errors on healthy volumes: the superblock free counters are not updated after format, and it expects refcount and snapshot sector checksums the runtime does not write. - Benchmark suite (`tests/benchmark`) and an `internal/metrics` package. - Go, Docker, and QEMU VM test suites, with `tests/test-manifest.yaml` and `make test-parity` to track coverage across them. @@ -53,6 +56,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - All nodes wrote the same journal region. A restarting node replayed and then reset the journal of nodes still writing, and no node could replay a crashed node's journal. Each node registry slot now owns a 4 MB journal, and a crashed node's journal is replayed through its slot. - Nodes never sent registry heartbeats, so a node mounting more than 10 seconds after another took the live node's registry slot. - After a partial write into a fallocated range, reads of the other fallocated blocks, and of the rest of the written block, returned old disk contents. They now return zeros. +- Snapshots and clones never copied on write: block refcounts stopped at 1, so a snapshot kept nothing and a write to a clone changed its source. Each snapshot now keeps the blocks it captured in a hidden inode, a write to a shared block copies it first, and a block is freed only when its last owner lets go. +- Deleting a snapshot after its file changed failed with a refcount underflow and left the snapshot deleting forever, because the delete used the file's current blocks. - Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge. - `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects. - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e77fd5d9..513f9202 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -266,12 +266,9 @@ func main() { log.Println("SFB: disabled (all files use full 1MB blocks)") } - // Create refcount region for snapshot CoW support - var refcountReg *refcount.Region - if *snapshotsEnabled { - refcountReg = refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) - log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) - } + // Refcounts stay on without snapshots: blocks shared earlier still need copy-on-write. + refcountReg := refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) + log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) @@ -307,7 +304,7 @@ func main() { // Initialize snapshot subsystem after verifying filesystem is valid if *snapshotsEnabled { - if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize); err != nil { + if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize, lockMgr, slotIdx); err != nil { log.Fatalf("Failed to initialize snapshots: %v", err) } log.Printf("Snapshots: enabled (registry at %dMB, %dMB capacity)", diff --git a/internal/extent/extent.go b/internal/extent/extent.go index 6ea3fab0..f40fc395 100644 --- a/internal/extent/extent.go +++ b/internal/extent/extent.go @@ -46,6 +46,9 @@ const ( FlagUnwritten = 1 << 1 // Allocated but lazy-zero (not yet written) FlagShared = 1 << 2 // Shared with snapshot (requires CoW) FlagSFB = 1 << 3 // Stored in SFB container (not full block) + // FlagCounted marks sharing that refcounts track. Older versions set + // FlagShared alone and never counted the owners. + FlagCounted = 1 << 6 ) // Hole is what LookupPhysical returns for a logical block with no data on disk. @@ -66,6 +69,9 @@ type CoWResult struct { OldPhysBlock uint64 // Original physical block (for refcount decrement) NewPhysBlock uint64 // New physical block after CoW WasShared bool // True if CoW was actually performed + // Uncounted means an older version shared the old block without counting + // its owners, so it must never be released. + Uncounted bool } // Extent represents a contiguous range of blocks. @@ -86,10 +92,18 @@ type Manager struct { leafCap int // extents per leaf before it splits indexCap int // leaves per tree + refs *refcount.Region // nil when snapshots are disabled + deferredMu sync.Mutex deferred []uint64 // frees postponed by lock order } +// SetRefcounts makes frees respect block sharing: a block shared by a +// snapshot or clone goes back to the bitmap only after its last owner frees it. +func (m *Manager) SetRefcounts(refs *refcount.Region) { + m.refs = refs +} + // NewManager creates a new extent manager. dataOffset is the device offset of // data block 0; extent tree nodes are stored in data blocks. func NewManager(dev *storage.Device, bm *bitmap.Region, lockMgr *lock.Manager, dataOffset uint64) *Manager { @@ -315,9 +329,8 @@ func (m *Manager) unmap(ctx context.Context, ino *inode.Inode, from, to uint64) // release updates block accounting for removed extents and queues their // blocks to be freed when the inode write commits: if that write loses a -// conflict, the inode on disk still maps them. Shared blocks stay allocated for -// the refcount system, and SFB extents point into containers rather than at -// bitmap blocks. +// conflict, the inode on disk still maps them. SFB extents point into +// containers rather than at bitmap blocks. func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Extent) { var blocks []uint64 for _, x := range removed { @@ -325,24 +338,42 @@ func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Exten if x.Flags&FlagUnwritten == 0 { ino.WrittenBlocks -= min(ino.WrittenBlocks, x.Length) } - if x.Flags&(FlagShared|FlagSFB) != 0 { - continue - } - for j := uint64(0); j < x.Length; j++ { - blocks = append(blocks, x.PhysicalBlock+j) + if x.Flags&FlagSFB == 0 { + blocks = append(blocks, m.ownedBlocks(x)...) } } if len(blocks) == 0 { return } - p := pendingOf(ino) - if p == nil { - p = &pendingTree{base: committedTree(ino)} - ino.PendingTree = p - } + p := ensurePending(ino) p.release = append(p.release, blocks...) } +// ownedBlocks returns the blocks of x that frees may give back. Nobody knows +// how many inodes share an uncounted block, so it stays allocated. +func (m *Manager) ownedBlocks(x Extent) []uint64 { + if x.Flags&FlagShared != 0 && (x.Flags&FlagCounted == 0 || m.refs == nil) { + return nil + } + blocks := make([]uint64, x.Length) + for j := range blocks { + blocks[j] = x.PhysicalBlock + uint64(j) + } + return blocks +} + +// HasUncountedSharing reports whether ino shares blocks the way older versions +// did, without refcounts. Such blocks cannot be shared again safely. +func (m *Manager) HasUncountedSharing(ino *inode.Inode) (bool, error) { + uncounted := false + err := m.forEach(ino, func(x Extent) { + if x.Flags&FlagShared != 0 && x.Flags&FlagCounted == 0 { + uncounted = true + } + }) + return uncounted, err +} + // Delete unmaps every block of an inode that is being freed and frees the // blocks now, since no inode write will follow to commit the change. func (m *Manager) Delete(ctx context.Context, ino *inode.Inode) error { @@ -529,7 +560,10 @@ func (m *Manager) CopyOnWrite(ctx context.Context, ino *inode.Inode, ext *Extent if err != nil { return nil, err } - result := &CoWResult{OldPhysBlock: cur.PhysicalBlock + offset} + result := &CoWResult{ + OldPhysBlock: cur.PhysicalBlock + offset, + Uncounted: cur.Flags&FlagShared != 0 && cur.Flags&FlagCounted == 0, + } block := Extent{logicalBlock, result.OldPhysBlock, 1, cur.Flags} needsCoW, err := m.NeedsCoW(ctx, &block, refcountReg) @@ -564,12 +598,17 @@ func (m *Manager) CopyOnWrite(ctx context.Context, ino *inode.Inode, ext *Extent return nil, fmt.Errorf("CoW write failed: %w", err) } - private := Extent{logicalBlock, newPhysBlock, 1, cur.Flags &^ FlagShared} + private := Extent{logicalBlock, newPhysBlock, 1, cur.Flags &^ (FlagShared | FlagCounted)} if err := m.remap(ctx, ino, private); err != nil { m.bitmap.Free(ctx, newPhysBlock) return nil, err } + // The caller releases the old block after the inode write, so that write + // must fail if another node copied the block first. + ensurePending(ino) *ext = private + // Readers keep cached block mappings until DataGeneration changes. + ino.DataGeneration++ result.NewPhysBlock = newPhysBlock result.WasShared = true @@ -599,7 +638,7 @@ func (m *Manager) MarkAllShared(ino *inode.Inode) error { if err != nil { return err } - if err := e.updateAll(func(x *Extent) { x.Flags |= FlagShared }); err != nil { + if err := e.updateAll(func(x *Extent) { x.Flags |= FlagShared | FlagCounted }); err != nil { return err } return e.commit() @@ -615,7 +654,7 @@ func (m *Manager) CopyExtentsForClone(ctx context.Context, src *inode.Inode, dst } all := []Extent{} if err := m.forEach(src, func(x Extent) { - x.Flags |= FlagShared + x.Flags |= FlagShared | FlagCounted all = append(all, x) }); err != nil { return err diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index 2f1f5040..90f7bc3c 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -2,6 +2,7 @@ package extent import ( "context" + "errors" "os" "path/filepath" "testing" @@ -1507,3 +1508,167 @@ func listExtents(tb testing.TB, mgr *Manager, ino *inode.Inode) []*Extent { } return extents } + +func TestFreeKeepsSharedBlocks(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + initial := env.bm.FreeCount() + + src := env.newInode() + if _, err := env.mgr.AllocateBlocks(env.ctx, src, 0, 2); err != nil { + t.Fatal(err) + } + if err := env.mgr.MarkAllShared(src); err != nil { + t.Fatal(err) + } + env.write(src) + clone := env.newInode() + if err := env.mgr.CopyExtentsForClone(env.ctx, src, clone); err != nil { + t.Fatal(err) + } + env.write(clone) + blocks, err := env.mgr.DataBlocks(src) + if err != nil { + t.Fatal(err) + } + if err := refs.Share(env.ctx, blocks); err != nil { + t.Fatal(err) + } + + truncate := func(ino *inode.Inode) { + t.Helper() + if err := env.mgr.Truncate(env.ctx, ino, 0); err != nil { + t.Fatal(err) + } + env.write(ino) + } + truncate(src) + if got := env.bm.FreeCount(); got != initial-2 { + t.Fatalf("free blocks after truncating the source = %d, want %d: the clone still maps them", got, initial-2) + } + truncate(clone) + if got := env.bm.FreeCount(); got != initial { + t.Fatalf("free blocks after truncating both = %d, want %d", got, initial) + } + +} + +// legacyShared maps a new block into each inode the way older versions shared +// blocks: FlagShared without FlagCounted, and a refcount of count. +func legacyShared(t *testing.T, env *treeEnv, refs *refcount.Region, count uint32, inodes ...*inode.Inode) uint64 { + t.Helper() + blocks, err := env.bm.Alloc(env.ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := refs.Set(env.ctx, blocks[0], count, refcount.FlagNone); err != nil { + t.Fatal(err) + } + for _, ino := range inodes { + env.insert(ino, Extent{LogicalBlock: 0, PhysicalBlock: blocks[0], Length: 1, Flags: FlagAllocated | FlagShared}) + env.write(ino) + } + return blocks[0] +} + +func TestUncountedSharingIsNeverFreed(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + truncate := func(ino *inode.Inode) { + t.Helper() + if err := env.mgr.Truncate(env.ctx, ino, 0); err != nil { + t.Fatal(err) + } + env.write(ino) + } + + // An old clone left a count of 0; an old snapshot and then an old clone left 1. + for _, count := range []uint32{0, 1} { + src, clone := env.newInode(), env.newInode() + block := legacyShared(t, env, refs, count, src, clone) + for _, ino := range []*inode.Inode{src, clone} { + uncounted, err := env.mgr.HasUncountedSharing(ino) + if err != nil || !uncounted { + t.Fatalf("count %d: HasUncountedSharing = %v, %v; want true", count, uncounted, err) + } + } + truncate(clone) + if ok, _ := env.bm.IsAllocated(block); !ok { + t.Fatalf("count %d: truncating the old clone freed block %d that its source maps", count, block) + } + } + + fresh := env.newInode() + if _, err := env.mgr.AllocateBlocks(env.ctx, fresh, 0, 1); err != nil { + t.Fatal(err) + } + if err := env.mgr.MarkAllShared(fresh); err != nil { + t.Fatal(err) + } + if uncounted, err := env.mgr.HasUncountedSharing(fresh); err != nil || uncounted { + t.Fatalf("sharing marked by this version reported as uncounted: %v, %v", uncounted, err) + } +} + +// Two nodes copy the same shared block of an inline file: only the first +// inode write may land, so the old block is released once. +func TestConcurrentInlineCopyOnWrite(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + + src := env.newInode() + ext, err := env.mgr.AllocateBlocks(env.ctx, src, 0, 1) + if err != nil { + t.Fatal(err) + } + old := ext.PhysicalBlock + if err := env.mgr.MarkAllShared(src); err != nil { + t.Fatal(err) + } + env.write(src) + if err := refs.Share(env.ctx, []uint64{old}); err != nil { + t.Fatal(err) + } + snap := env.newInode() + if err := env.mgr.CopyExtentsForClone(env.ctx, src, snap); err != nil { + t.Fatal(err) + } + env.write(snap) + + nodeA, nodeB := env.reread(src), env.reread(src) + var copies []uint64 + for _, ino := range []*inode.Inode{nodeA, nodeB} { + x, _, err := env.mgr.Lookup(ino, 0) + if err != nil { + t.Fatal(err) + } + res, err := env.mgr.CopyOnWrite(env.ctx, ino, x, 0, treeTestDataOffset, refs) + if err != nil || !res.WasShared { + t.Fatalf("copy-on-write: %+v, %v", res, err) + } + copies = append(copies, res.NewPhysBlock) + } + + env.write(nodeA) + env.mgr.ReleaseBlocks(env.ctx, []uint64{old}) + if err := env.tbl.Write(env.ctx, nodeB); !errors.Is(err, ErrTreeConflict) { + t.Fatalf("second copy's inode write: got %v, want ErrTreeConflict", err) + } + env.mgr.DiscardPending(env.ctx, nodeB) + if err := env.bm.Free(env.ctx, copies[1]); err != nil { + t.Fatal(err) + } + + if ok, _ := env.bm.IsAllocated(old); !ok { + t.Fatalf("block %d freed while the snapshot still maps it", old) + } + if e, err := refs.Get(env.ctx, old); err != nil || e.RefCount != 1 { + t.Fatalf("old block refcount = %+v, %v; want 1 (the snapshot)", e, err) + } + if got, _, err := env.mgr.Lookup(env.reread(src), 0); err != nil || got.PhysicalBlock != copies[0] { + t.Fatalf("file maps %+v, %v; want the first copy %d", got, err, copies[0]) + } +} diff --git a/internal/extent/tree_edit.go b/internal/extent/tree_edit.go index e4a950ec..432cddd9 100644 --- a/internal/extent/tree_edit.go +++ b/internal/extent/tree_edit.go @@ -714,28 +714,64 @@ func (m *Manager) allocTreeBlocks(ctx context.Context, n int) ([]uint64, error) return out, nil } -// freeBlocks returns blocks to the bitmap. A caller holding an inode lock -// cannot take the bitmap lock, so those frees wait for the next freeBlocks -// call made without it. +// ensurePending returns ino's pending change, creating an empty one if needed. +// Any pending change makes the inode write check that the extents on disk are +// still the ones this change started from. +func ensurePending(ino *inode.Inode) *pendingTree { + p := pendingOf(ino) + if p == nil { + p = &pendingTree{base: committedTree(ino)} + ino.PendingTree = p + } + return p +} + +// ReleaseBlocks drops one owner from each block and frees the blocks that no +// owner holds any more, such as a block a write just copied away from. +func (m *Manager) ReleaseBlocks(ctx context.Context, blocks []uint64) { + m.freeBlocks(ctx, blocks) +} + +// freeBlocks drops one owner from each block and returns the blocks no owner +// holds to the bitmap. A caller holding an inode lock cannot take the refcount +// or bitmap lock, so those frees wait for the next freeBlocks call made +// without it. func (m *Manager) freeBlocks(ctx context.Context, blocks []uint64) { m.deferredMu.Lock() blocks = append(blocks, m.deferred...) m.deferred = nil m.deferredMu.Unlock() + if m.refs != nil && len(blocks) > 0 { + free, err := m.refs.Release(ctx, blocks) + switch { + case errors.Is(err, lock.ErrDeadlockRisk): + m.deferFree(blocks...) + return + case err != nil: + log.Printf("[extent] release %d blocks: %v; leaving them allocated", len(blocks), err) + return + } + blocks = free + } + for _, b := range blocks { err := m.bitmap.Free(ctx, b) switch { case errors.Is(err, lock.ErrDeadlockRisk): - m.deferredMu.Lock() - m.deferred = append(m.deferred, b) - m.deferredMu.Unlock() + m.deferFree(b) case err != nil: log.Printf("[extent] free block %d: %v", b, err) } } } +func (m *Manager) deferFree(blocks ...uint64) { + m.deferredMu.Lock() + m.deferred = append(m.deferred, blocks...) + m.deferredMu.Unlock() +} + // DiscardPending drops ino's uncommitted extent-tree change after a failed // inode write, freeing the tree blocks allocated for it. Blocks the change // would have released stay allocated because the inode on disk may use them. diff --git a/internal/fsck/fsck.go b/internal/fsck/fsck.go index 74ccd8a9..566cc781 100644 --- a/internal/fsck/fsck.go +++ b/internal/fsck/fsck.go @@ -334,6 +334,7 @@ type Checker struct { dirEntryRefs map[uint32]int // inode -> reference count from dir entries refcountEntries map[uint64]uint32 // block -> refcount from table snapshotInodes map[uint32]bool // inodes referenced by snapshots + snapshotData map[uint32]bool // inodes holding a snapshot's captured extents extentTrees map[uint32]*extentTree } @@ -401,6 +402,7 @@ func New(dev *storage.Device, opts Options) *Checker { dirEntryRefs: make(map[uint32]int), refcountEntries: make(map[uint64]uint32), snapshotInodes: make(map[uint32]bool), + snapshotData: make(map[uint32]bool), extentTrees: make(map[uint32]*extentTree), } } diff --git a/internal/fsck/phase3_inodes.go b/internal/fsck/phase3_inodes.go index f19da2cb..d69cb220 100644 --- a/internal/fsck/phase3_inodes.go +++ b/internal/fsck/phase3_inodes.go @@ -40,6 +40,7 @@ const ( FlagHasExtentTree uint16 = 1 << 0 FlagHasSnapshot uint16 = 1 << 1 FlagDeleted uint16 = 1 << 2 + FlagSnapshotData uint16 = 1 << 6 ) // InodeData holds parsed inode information. @@ -222,6 +223,10 @@ func (c *Checker) checkInodes() error { true) } + if inode.Flags&FlagSnapshotData != 0 { + c.snapshotData[inodeNum] = true + } + // Record extents for cross-validation c.recordInodeExtents(inodeNum, inode) } diff --git a/internal/fsck/phase5_refcounts.go b/internal/fsck/phase5_refcounts.go index 8661b77c..5b6a3fb3 100644 --- a/internal/fsck/phase5_refcounts.go +++ b/internal/fsck/phase5_refcounts.go @@ -40,8 +40,8 @@ const ( // // Checks performed: // - Each sector has valid CRC32C checksum -// - RefCount > 0 for blocks marked allocated in bitmap -// - RefCount == 0 for free blocks +// - RefCount == 0 for free blocks (0 or 1 means one owner, so an +// allocated block may have either) // - RefCount > 1 implies FlagShared should be set in extent // - No refcount overflow (0xFFFFFFFF) // - Cross-validate with extent references from Phase 4 @@ -133,34 +133,16 @@ func (c *Checker) checkRefcounts() error { isAllocatedInBitmap := c.bitmapAllocated[blockNum] refsFromInodes := c.inodeBlocks[blockNum] - if isAllocatedInBitmap { - // Allocated block should have refcount >= 1 - if entry.RefCount == 0 { - c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - "block allocated in bitmap but refcount is 0", - true) - inconsistentEntries++ - } - } else { - // Free block should have refcount == 0 - if entry.RefCount != 0 { - c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - fmt.Sprintf("block free in bitmap but refcount is %d", entry.RefCount), - true) - inconsistentEntries++ - } - } - - // Cross-validate with inode references - if refsFromInodes > 0 && entry.RefCount == 0 { + if !isAllocatedInBitmap && entry.RefCount != 0 { c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - fmt.Sprintf("referenced by %d inode(s) but refcount is 0", refsFromInodes), + fmt.Sprintf("block free in bitmap but refcount is %d", entry.RefCount), true) + inconsistentEntries++ } - // Check for shared blocks without proper refcount + // An undercounted block is freed while another inode still maps it. if refsFromInodes > 1 && entry.RefCount < uint32(refsFromInodes) { - c.addProblem(PhaseRefcounts, Warning, fmt.Sprintf("block %d", blockNum), + c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), fmt.Sprintf("referenced by %d inodes but refcount is only %d", refsFromInodes, entry.RefCount), true) } diff --git a/internal/fsck/phase6_directories.go b/internal/fsck/phase6_directories.go index fda44c05..76f08bef 100644 --- a/internal/fsck/phase6_directories.go +++ b/internal/fsck/phase6_directories.go @@ -148,8 +148,8 @@ func (c *Checker) checkDirectories() error { // Check for orphaned inodes (in use but not referenced by any directory) for inodeNum := range c.inodeUsed { - if inodeNum == InodeRoot { - continue // Root directory is special + if inodeNum == InodeRoot || c.snapshotData[inodeNum] { + continue } if c.dirEntryRefs[inodeNum] == 0 { orphanedInodes++ diff --git a/internal/fsck/phase7_snapshots.go b/internal/fsck/phase7_snapshots.go index aaf71769..7410bb7e 100644 --- a/internal/fsck/phase7_snapshots.go +++ b/internal/fsck/phase7_snapshots.go @@ -23,6 +23,9 @@ const ( // SnapshotMagic identifies a valid snapshot descriptor. SnapshotMagic uint32 = 0x534E4150 // "SNAP" + // SnapshotFlagHasData marks descriptors whose DataInode holds the captured extents. + SnapshotFlagHasData uint8 = 1 << 4 + // MaxSnapshotNameLen is the maximum length of a snapshot name. MaxSnapshotNameLen = 48 ) @@ -37,20 +40,20 @@ const ( // SnapshotDescriptor holds parsed snapshot information. type SnapshotDescriptor struct { - Magic uint32 - SnapshotID uint64 - ParentID uint64 - TargetInode uint32 - State uint8 - Flags uint8 - ExtentTreeRoot uint64 - TotalBlocks uint64 - TotalSize uint64 - CreateTime int64 - DeleteTime int64 - RefCountJobID uint64 - Name string - Checksum uint32 + Magic uint32 + SnapshotID uint64 + ParentID uint64 + TargetInode uint32 + State uint8 + Flags uint8 + DataInode uint32 + TotalBlocks uint64 + TotalSize uint64 + CreateTime int64 + DeleteTime int64 + RefCountJobID uint64 + Name string + Checksum uint32 } // checkSnapshots validates the snapshot registry (Phase 7). @@ -99,6 +102,7 @@ func (c *Checker) checkSnapshots() error { // Track snapshot IDs and names for duplicate detection seenIDs := make(map[uint64]bool) seenNames := make(map[string]uint64) // name -> snapshot ID + dataInodes := make(map[uint32]bool) for sectorIdx := uint64(0); sectorIdx < totalSectors; sectorIdx++ { sectorOffset := snapshotOffset + sectorIdx*SnapshotSectorSize @@ -197,6 +201,16 @@ func (c *Checker) checkSnapshots() error { c.snapshotInodes[desc.TargetInode] = true } + if desc.Flags&SnapshotFlagHasData != 0 && desc.DataInode != 0 { + if !c.snapshotData[desc.DataInode] { + c.addProblem(PhaseSnapshots, Error, fmt.Sprintf("snapshot %d", desc.SnapshotID), + fmt.Sprintf("data inode %d is not a snapshot data inode", desc.DataInode), + false) + } + dataInodes[desc.DataInode] = true + c.snapshotInodes[desc.DataInode] = true + } + // Validate parent reference (if not root snapshot) if desc.ParentID != 0 && !seenIDs[desc.ParentID] { // Parent might come later - this is a forward reference issue @@ -220,6 +234,14 @@ func (c *Checker) checkSnapshots() error { } } + for num := range c.snapshotData { + if !dataInodes[num] { + c.addProblem(PhaseSnapshots, Warning, fmt.Sprintf("inode %d", num), + "snapshot data inode belongs to no snapshot (left by a crash during snapshot creation)", + true) + } + } + // Compare with superblock counter if activeSnapshots != expectedCount { c.addProblem(PhaseSnapshots, Error, "snapshot count", @@ -247,7 +269,7 @@ func parseSnapshotDescriptor(buf []byte) *SnapshotDescriptor { desc.State = buf[24] desc.Flags = buf[25] // Bytes 26-27 reserved - desc.ExtentTreeRoot = binary.LittleEndian.Uint64(buf[28:36]) + desc.DataInode = binary.LittleEndian.Uint32(buf[28:32]) desc.TotalBlocks = binary.LittleEndian.Uint64(buf[36:44]) desc.TotalSize = binary.LittleEndian.Uint64(buf[44:52]) desc.CreateTime = int64(binary.LittleEndian.Uint64(buf[52:60])) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 9a524dfa..140f756d 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -42,6 +42,10 @@ import ( // modification was detected and the Write operation should be retried. var errRetryWrite = errors.New("retry write operation") +// beforeCoWCommit lets tests run another node's write after a copy-on-write, +// before this write takes the inode lock and commits the copy. +var beforeCoWCommit func() + // MaxWriteRetries is the maximum number of retry attempts for Write operations // when concurrent modifications are detected. const MaxWriteRetries = 3 @@ -202,6 +206,9 @@ func NewWithComponents( nodeUUID [16]byte, dataOffset uint64, ) *FS { + if refcountReg != nil { + extentMgr.SetRefcounts(refcountReg) + } return &FS{ dev: dev, lockMgr: lockMgr, @@ -226,15 +233,19 @@ func NewWithComponents( // Parameters: // - snapshotOffset: byte offset where snapshot region starts on device // - snapshotSize: size of snapshot region in bytes +// - lockMgr: serializes registry changes across nodes (nil for single-node use) +// - nodeSlot: this mount's node registry slot, which keeps snapshot IDs and +// descriptor sectors apart from other nodes' even without the lock // // This must be called before any snapshot operations can succeed. -func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64) error { +func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64, lockMgr *lock.Manager, nodeSlot int) error { if f.refcountReg == nil { return fmt.Errorf("refcount region must be initialized before snapshots") } // Create snapshot registry - f.snapshotReg = snapshot.NewRegistry(f.dev, snapshotOffset, snapshotSize, f.lockMgr) + f.snapshotReg = snapshot.NewRegistry(f.dev, snapshotOffset, snapshotSize, lockMgr) + f.snapshotReg.SetNodeSlot(nodeSlot) // Scan registry to rebuild state stats, err := f.snapshotReg.Scan(ctx) @@ -246,7 +257,7 @@ func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uin // Create and start background worker cfg := snapshot.DefaultWorkerConfig() - f.snapshotWkr = snapshot.NewWorker(f.snapshotReg, f.refcountReg, f.extentMgr, f.inodeTbl, cfg) + f.snapshotWkr = snapshot.NewWorker(f.snapshotReg, f.extentMgr, f.inodeTbl, cfg) f.snapshotWkr.Start() // Recover any incomplete jobs from crash @@ -2356,76 +2367,59 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } handleExtent: - // === Copy-on-Write for Shared Blocks (Snapshot Support) === - // If this extent is shared with a snapshot (FlagShared + RefCount > 1), - // we must allocate a new block before writing to preserve snapshot data. + // A shared block is copied before the write, so its other owners keep their data. if ext.Flags&extent.FlagShared != 0 && f.fs.refcountReg != nil { - // Check if CoW is actually needed (refcount > 1) + // Refcounts and the bitmap come before the inode in lock order. + if err := persistAndUnlock(); err != nil { + return 0, err + } needsCoW, cowErr := f.fs.extentMgr.NeedsCoW(ctx, blockExtent(ext, extOffset), f.fs.refcountReg) if cowErr != nil { log.Printf("[DEBUG] Write inode %d: CoW check failed for block %d: %v", f.inodeNum, ext.PhysicalBlock, cowErr) return 0, syscall.EIO } - if needsCoW { - // Need exclusive inode lock for CoW (modifies extent tree) - if !lockHeld && f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to acquire lock for CoW: %v", f.inodeNum, err) - return 0, syscall.EIO - } - lockHeld = true - - // Re-read inode after acquiring lock - freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: re-read for CoW failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - if freshIno.MetaGeneration != ino.MetaGeneration { - log.Printf("[DEBUG] Write inode %d: MetaGeneration changed during CoW", f.inodeNum) + cowResult, cowErr := f.fs.extentMgr.CopyOnWrite(ctx, ino, ext, blockNum, f.fs.dataOffset, f.fs.refcountReg) + if cowErr != nil { + log.Printf("[DEBUG] Write inode %d: CoW failed for block %d: %v", + f.inodeNum, ext.PhysicalBlock, cowErr) + f.fs.extentMgr.DiscardPending(ctx, ino) + // The unlocked copy may point at tree nodes another node has since rewritten. + if errors.Is(cowErr, extent.ErrStaleTree) { return 0, errRetryWrite } - ino = freshIno - - // Re-lookup extent - another node may have already done CoW - ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: extent lookup for CoW failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - - // Re-check if CoW is still needed (another node may have done it) - needsCoW, cowErr = f.fs.extentMgr.NeedsCoW(ctx, blockExtent(ext, extOffset), f.fs.refcountReg) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: CoW re-check failed: %v", f.inodeNum, cowErr) - return 0, syscall.EIO - } + return 0, syscall.EIO } - - // Perform CoW if still needed - if needsCoW { - cowResult, cowErr := f.fs.extentMgr.CopyOnWrite(ctx, ino, ext, blockNum, f.fs.dataOffset, f.fs.refcountReg) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: CoW failed for block %d: %v", - f.inodeNum, ext.PhysicalBlock, cowErr) - return 0, syscall.EIO + if cowResult.WasShared { + if beforeCoWCommit != nil { + beforeCoWCommit() } - - if cowResult.WasShared { - inoDirty = true - log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", - f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) - - cowDone = append(cowDone, cowResult) - - // Re-lookup extent to get updated physical block - ext, extOffset, cowErr = f.fs.extentMgr.Lookup(ino, blockNum) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: extent lookup after CoW failed: %v", f.inodeNum, cowErr) + // Copied on an unlocked copy: keep it only if no node wrote the inode meanwhile. + if f.fs.lockMgr != nil { + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + log.Printf("[DEBUG] Write inode %d: failed to acquire lock for CoW: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + f.fs.extentMgr.FreePhysicalBlock(ctx, cowResult.NewPhysBlock) return 0, syscall.EIO } + lockHeld = true + freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || freshIno.MetaGeneration != ino.MetaGeneration { + log.Printf("[DEBUG] Write inode %d: inode changed during CoW (err=%v)", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + f.fs.extentMgr.FreePhysicalBlock(ctx, cowResult.NewPhysBlock) + return 0, errRetryWrite + } + } + inoDirty = true + cowDone = append(cowDone, cowResult) + log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", + f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) + if ext, extOffset, cowErr = f.fs.extentMgr.Lookup(ino, blockNum); cowErr != nil { + log.Printf("[DEBUG] Write inode %d: extent lookup after CoW failed: %v", f.inodeNum, cowErr) + return 0, syscall.EIO } } } @@ -2641,6 +2635,20 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i ino.Size = newSize ino.Mtime = newMtime + // releaseCopied drops this file's ownership of blocks it copied away from. + // Blocks an older version shared without counting stay allocated. + releaseCopied := func(cows []*extent.CoWResult) { + var old []uint64 + for _, cow := range cows { + if !cow.Uncounted { + old = append(old, cow.OldPhysBlock) + } + } + if len(old) > 0 { + f.fs.extentMgr.ReleaseBlocks(ctx, old) + } + } + // On a tree conflict the blocks allocated since the last mid-write persist are // unreferenced. Close the intent first so journal replay cannot map them again, // then free them. Blocks the persist committed stay mapped on disk. @@ -2672,12 +2680,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } // The retry finds these blocks already copied, so release the old ones now. - for _, cow := range cowDone[:committedCoW] { - if _, err := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock); err != nil { - log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", - f.inodeNum, cow.OldPhysBlock, err) - } - } + releaseCopied(cowDone[:committedCoW]) return errRetryWrite } @@ -2686,7 +2689,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return 0, writeFailed(err) } } - // Write inode - use WriteWithLockHeld if we hold the lock if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { @@ -2704,17 +2706,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseInodeLock() - for _, cow := range cowDone { - newCount, decErr := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock) - if decErr != nil { - // Log but don't fail - the write succeeded, refcount is secondary - log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", - f.inodeNum, cow.OldPhysBlock, decErr) - } else if newCount == 0 { - log.Printf("[DEBUG] Write inode %d: old block %d refcount reached 0 (can be freed)", - f.inodeNum, cow.OldPhysBlock) - } - } + releaseCopied(cowDone) // Cache the freshly-written inode for subsequent reads f.fs.cacheInode(ino) @@ -3239,6 +3231,8 @@ func (f *File) allocate(ctx context.Context, off uint64, size uint64, mode uint3 // Used for snapshot creation via user.snapshot attribute and clone creation via user.clone. func (f *File) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno { defer f.fs.lockInode(f.inodeNum)() + // A snapshot or clone spans several writes; an interrupted caller must not stop it halfway. + ctx = context.WithoutCancel(ctx) switch attr { case "user.snapshot": return f.createSnapshot(ctx, string(data)) @@ -3277,15 +3271,15 @@ func (f *File) Removexattr(ctx context.Context, attr string) syscall.Errno { if snapName == "" { return syscall.EINVAL } - return f.deleteSnapshot(ctx, snapName) + return f.deleteSnapshot(context.WithoutCancel(ctx), snapName) } return syscall.ENODATA } // markShared marks every extent shared so writes to either copy use CoW. -func (f *File) markShared(ino *inode.Inode) error { - if err := f.fs.extentMgr.MarkAllShared(ino); err != nil { +func (f *FS) markShared(ino *inode.Inode) error { + if err := f.extentMgr.MarkAllShared(ino); err != nil { return err } ino.Flags |= inode.FlagHasSnapshot @@ -3293,29 +3287,162 @@ func (f *File) markShared(ino *inode.Inode) error { return nil } -// createSnapshot creates a new snapshot of this file. +// errUncountedSharing means the file shares blocks the way older versions did, +// without refcounts, so its blocks cannot be shared again safely. +var errUncountedSharing = errors.New("an older version shared this file's blocks without refcounts; run goclusterfs-fsck, and copy the file to snapshot or clone it") + +// shareInto gives dst the extents of inode srcNum, sharing its data blocks. +// src is marked shared first, then each block gains an owner, durably, before +// dst maps it; a crash in between leaves src copying blocks it alone owns. +// stillValid, if set, runs after the blocks gain owners and before dst maps +// them. On error the caller discards dst, which gives back any blocks it maps. +func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, stillValid func() error) error { + src, err := f.inodeTbl.Read(srcNum) + if err != nil { + return err + } + uncounted, err := f.extentMgr.HasUncountedSharing(src) + if err != nil { + return err + } + if uncounted { + return errUncountedSharing + } + // A snapshot's data inode is marked shared when it is created and never written again. + if src.Flags&inode.FlagSnapshotData == 0 { + if src, err = f.updateInode(ctx, srcNum, f.markShared); err != nil { + return err + } + } + blocks, err := f.extentMgr.DataBlocks(src) + if err != nil { + return err + } + if err := f.refcountReg.Share(ctx, blocks); err != nil { + return err + } + err = f.dev.Sync() + if err == nil && stillValid != nil { + err = stillValid() + } + if err != nil { + f.extentMgr.ReleaseBlocks(ctx, blocks) + return err + } + if err := f.extentMgr.CopyExtentsForClone(ctx, src, dst); err != nil { + f.extentMgr.ReleaseBlocks(ctx, blocks) + return err + } + dst.Size = src.Size + if err := f.inodeTbl.Write(ctx, dst); err != nil { + f.extentMgr.DiscardPending(ctx, dst) + // Only a conflict proves dst on disk does not map the blocks. + if errors.Is(err, extent.ErrTreeConflict) { + f.extentMgr.ReleaseBlocks(ctx, blocks) + } + return err + } + return nil +} + +// sharingErrno maps a failure to share a file's blocks to an errno. +func sharingErrno(err error) syscall.Errno { + if errors.Is(err, errUncountedSharing) { + return syscall.ENOTSUP + } + return syscall.EIO +} + +// claimInode allocates an inode slot, retrying when another node takes the +// free slot first. +func (f *FS) claimInode(ctx context.Context) (*inode.Inode, error) { + for attempt := 0; ; attempt++ { + num, err := f.inodeTbl.FindFreeSlot() + if err != nil { + return nil, err + } + ino, err := f.inodeTbl.ClaimSlot(ctx, num) + if !errors.Is(err, inode.ErrInodeInUse) || attempt == maxConflictRetries-1 { + return ino, err + } + } +} + +// discardInode gives back the blocks an unlinked inode maps and frees it. The +// empty extent map is committed before any block is released. +func (f *FS) discardInode(ctx context.Context, num uint32) { + _, err := f.updateInode(ctx, num, func(ino *inode.Inode) error { + return f.extentMgr.Truncate(ctx, ino, 0) + }) + if err == nil { + err = f.inodeTbl.Free(ctx, num) + } + if err != nil { + log.Printf("[clone] Failed to discard inode %d: %v", num, err) + } +} + +// dropSnapshot marks a snapshot deleting and queues the job that gives back +// its blocks. A snapshot left deleting is finished by the next mount. +func (f *FS) dropSnapshot(ctx context.Context, snapshotID uint64) error { + if err := f.snapshotReg.Delete(ctx, snapshotID); err != nil { + return err + } + if _, err := f.snapshotWkr.QueueDeleteJob(ctx, snapshotID); err != nil { + log.Printf("[snapshot] Delete of snapshot %d waits for the next mount: %v", snapshotID, err) + } + return nil +} + +// sharableInode reads this file's inode, moving it out of an SFB container +// first: container slots are written in place and cannot be shared. +func (f *File) sharableInode(ctx context.Context) (*inode.Inode, error) { + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || !ino.IsSFBBacked() { + return ino, err + } + if err := f.migrateToExtent(ctx, ino); err != nil { + return nil, err + } + return f.fs.inodeTbl.Read(f.inodeNum) +} + +// createSnapshot creates a new snapshot of this file. The snapshot's data is a +// hidden inode that shares the file's blocks as they are now. func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { if f.fs.snapshotReg == nil || f.fs.snapshotWkr == nil { log.Printf("[DEBUG] createSnapshot: snapshot subsystem not initialized") return syscall.ENOTSUP } + if len(name) >= snapshot.MaxSnapshotNameLen { + return syscall.ENAMETOOLONG + } - // Mark all extents as shared BEFORE creating snapshot (CoW active immediately) - ino, err := f.fs.updateInode(ctx, f.inodeNum, f.markShared) + src, err := f.sharableInode(ctx) if err != nil { - log.Printf("[snapshot] Failed to mark extents shared for inode %d: %v", f.inodeNum, err) + log.Printf("[snapshot] Failed to read inode %d: %v", f.inodeNum, err) return syscall.EIO } - blockNums, err := f.fs.extentMgr.DataBlocks(ino) + data, err := f.fs.claimInode(ctx) if err != nil { - log.Printf("[snapshot] Failed to list blocks for inode %d: %v", f.inodeNum, err) + log.Printf("[snapshot] No inode for the data of a snapshot of inode %d: %v", f.inodeNum, err) + return syscall.ENOSPC + } + data.Mode, data.UID, data.GID = src.Mode, src.UID, src.GID + data.Mtime, data.Ctime = src.Mtime, src.Ctime + data.Flags |= inode.FlagSnapshotData + if err := f.fs.inodeTbl.Write(ctx, data); err != nil { + f.fs.inodeTbl.Free(ctx, data.InodeNum) + log.Printf("[snapshot] Failed to write data inode %d: %v", data.InodeNum, err) return syscall.EIO } desc, err := f.fs.snapshotReg.Create(ctx, name, f.inodeNum, 0, - ino.ExtentTreeRoot, uint64(len(blockNums)), ino.Size, snapshot.FlagReadOnly) + data.InodeNum, src.AllocatedBlocks, src.Size, snapshot.FlagReadOnly) if err != nil { + log.Printf("[snapshot] Failed to add '%s' to the registry: %v", name, err) + f.fs.inodeTbl.Free(ctx, data.InodeNum) if errors.Is(err, snapshot.ErrSnapshotExists) { return syscall.EEXIST } @@ -3325,9 +3452,16 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EIO } - // Queue background refcount increment (returns immediately) - if _, err := f.fs.snapshotWkr.QueueCreateJob(ctx, desc.SnapshotID, f.inodeNum, blockNums); err != nil { - log.Printf("[snapshot] Warning: queue job failed: %v", err) + err = f.fs.shareInto(ctx, f.inodeNum, data, nil) + if err == nil { + err = f.fs.snapshotReg.Activate(ctx, desc.SnapshotID) + } + if err != nil { + log.Printf("[snapshot] Failed to create '%s' for inode %d: %v", name, f.inodeNum, err) + if dropErr := f.fs.dropSnapshot(ctx, desc.SnapshotID); dropErr != nil { + log.Printf("[snapshot] Failed to roll back snapshot %d: %v", desc.SnapshotID, dropErr) + } + return sharingErrno(err) } log.Printf("[snapshot] Created '%s' (ID=%d) for inode %d", name, desc.SnapshotID, f.inodeNum) @@ -3352,21 +3486,11 @@ func (f *File) deleteSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EINVAL } - if err := f.fs.snapshotReg.Delete(ctx, desc.SnapshotID); err != nil { + if err := f.fs.dropSnapshot(ctx, desc.SnapshotID); err != nil { + log.Printf("[snapshot] Failed to delete '%s': %v", name, err) return syscall.EIO } - var blockNums []uint64 - if ino, err := f.fs.inodeTbl.Read(f.inodeNum); err == nil { - if blockNums, err = f.fs.extentMgr.DataBlocks(ino); err != nil { - log.Printf("[snapshot] Failed to list blocks for inode %d: %v", f.inodeNum, err) - } - } - - if _, err := f.fs.snapshotWkr.QueueDeleteJob(ctx, desc.SnapshotID, blockNums); err != nil { - log.Printf("[snapshot] Warning: queue delete job failed: %v", err) - } - log.Printf("[snapshot] Deleting '%s' (ID=%d)", name, desc.SnapshotID) return 0 } @@ -3376,6 +3500,19 @@ func (f *File) deleteSnapshot(ctx context.Context, name string) syscall.Errno { // point Copy-on-Write allocates new blocks for the modified data. func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { log.Printf("[clone] Creating clone of inode %d at %s", f.inodeNum, destPath) + if _, err := f.sharableInode(ctx); err != nil { + log.Printf("[clone] Failed to read inode %d: %v", f.inodeNum, err) + return syscall.EIO + } + return f.cloneInode(ctx, f.inodeNum, destPath, nil) +} + +// cloneInode creates a file at destPath that shares the blocks of inode srcNum. +// stillValid is passed to shareInto. +func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string, stillValid func() error) syscall.Errno { + if f.fs.refcountReg == nil { + return syscall.ENOTSUP + } // Parse destination path parentPath, cloneName, err := parseClonePath(destPath) @@ -3403,10 +3540,9 @@ func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { return syscall.EEXIST } - // Mark source extents as shared (enables CoW on source) - srcIno, err := f.fs.updateInode(ctx, f.inodeNum, f.markShared) + srcIno, err := f.fs.inodeTbl.Read(srcNum) if err != nil { - log.Printf("[clone] Failed to update source inode: %v", err) + log.Printf("[clone] Failed to read source inode %d: %v", srcNum, err) return syscall.EIO } @@ -3435,47 +3571,24 @@ func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { return syscall.ENOSPC } - // Copy extents with FlagShared set - if err := f.fs.extentMgr.CopyExtentsForClone(ctx, srcIno, cloneIno); err != nil { - f.fs.dirMgr.Remove(ctx, rootIno, cloneName) - f.fs.inodeTbl.Free(ctx, cloneInodeNum) - log.Printf("[clone] Failed to copy extents: %v", err) - return syscall.EIO - } - - // Copy attributes from source now := time.Now().UnixNano() cloneIno.Mode = srcIno.Mode cloneIno.UID = srcIno.UID cloneIno.GID = srcIno.GID - cloneIno.Size = srcIno.Size cloneIno.LinkCount = 1 cloneIno.Flags |= inode.FlagHasSnapshot cloneIno.Atime = now cloneIno.Mtime = now cloneIno.Ctime = now - // Write clone inode - if err := f.fs.inodeTbl.Write(ctx, cloneIno); err != nil { - f.fs.extentMgr.DiscardPending(ctx, cloneIno) + if err := f.fs.shareInto(ctx, srcNum, cloneIno, stillValid); err != nil { f.fs.dirMgr.Remove(ctx, rootIno, cloneName) - f.fs.inodeTbl.Free(ctx, cloneInodeNum) - log.Printf("[clone] Failed to write clone inode: %v", err) - return syscall.EIO - } - - // Queue background refcount increment for shared blocks - blockNums, err := f.fs.extentMgr.DataBlocks(srcIno) - if err != nil { - log.Printf("[clone] Failed to list source blocks: %v", err) - } - if len(blockNums) > 0 && f.fs.snapshotWkr != nil { - if _, err := f.fs.snapshotWkr.QueueCreateJob(ctx, 0, f.inodeNum, blockNums); err != nil { - log.Printf("[clone] Warning: queue refcount job failed: %v", err) - } + f.fs.discardInode(ctx, cloneInodeNum) + log.Printf("[clone] Failed to share blocks of inode %d: %v", srcNum, err) + return sharingErrno(err) } - log.Printf("[clone] Created '%s' (inode %d) from inode %d", cloneName, cloneInodeNum, f.inodeNum) + log.Printf("[clone] Created '%s' (inode %d) from inode %d", cloneName, cloneInodeNum, srcNum) return 0 } @@ -3507,8 +3620,23 @@ func (f *File) createCloneFromSnapshot(ctx context.Context, data string) syscall log.Printf("[clone] Snapshot '%s' belongs to inode %d, not %d", snapName, desc.TargetInode, f.inodeNum) return syscall.EINVAL } + if desc.State != snapshot.StateActive { + return syscall.EBUSY + } + if !desc.HasData() { + log.Printf("[clone] Snapshot '%s' was made by an older version and holds no data", snapName) + return syscall.ENOTSUP + } - return f.createClone(ctx, destPath) + // A delete that starts before the blocks gain their new owner may free them. + stillActive := func() error { + d, err := f.fs.snapshotReg.Get(ctx, desc.SnapshotID) + if err == nil && d.State != snapshot.StateActive { + err = snapshot.ErrSnapshotInvalidState + } + return err + } + return f.cloneInode(ctx, desc.DataInode, destPath, stillActive) } // listSnapshots returns snapshot names for this file. diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go new file mode 100644 index 00000000..4469e2e8 --- /dev/null +++ b/internal/goclusterfs/snapshot_test.go @@ -0,0 +1,668 @@ +package goclusterfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/hanwen/go-fuse/v2/fuse" + + "github.com/supporttools/GoClusterFS/internal/bitmap" + "github.com/supporttools/GoClusterFS/internal/dir" + "github.com/supporttools/GoClusterFS/internal/extent" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/refcount" + "github.com/supporttools/GoClusterFS/internal/snapshot" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +const ( + snapFSBitmapOffset = 1 << 20 + snapFSInodeOffset = 2 << 20 + snapFSRefcountOffset = 3 << 20 + snapFSRegistryOffset = 3<<20 + 64<<10 + snapFSRegistrySize = 64 << 10 + snapFSLockOffset = 3<<20 + 256<<10 + snapFSLockSize = 512 << 10 + snapFSDataOffset = 4 << 20 + snapFSDataBlocks = 60 +) + +// snapFSNodes hands each opened FS its own node slot, so instances on one +// image lock against each other like separate nodes. +var snapFSNodes atomic.Int32 + +func newSnapshotFSImage(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fs.img") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(path, snapFSDataOffset+snapFSDataBlocks*BlockSize); err != nil { + t.Fatal(err) + } + return path +} + +// openSnapshotFS mounts an image with snapshots and the distributed locks +// enabled, as a fresh node would. +func openSnapshotFS(t *testing.T, path string) *FS { + t.Helper() + dev, err := storage.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { dev.Close() }) + + slot := int(snapFSNodes.Add(1)-1) % 64 + var nodeUUID [16]byte + copy(nodeUUID[:], "testnode12345678") + nodeUUID[15] = byte(slot) + locks := lock.NewManager(dev, snapFSLockOffset, snapFSLockSize, nodeUUID, slot) + bm := bitmap.NewRegion(dev, snapFSBitmapOffset, snapFSDataBlocks, locks, nodeUUID) + inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, locks, nodeUUID) + extentMgr := extent.NewManager(dev, bm, locks, snapFSDataOffset) + inodeTbl.SetTreeFlusher(extentMgr) + refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, locks) + dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, snapFSDataOffset) + fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, snapFSDataOffset) + if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, locks, slot); err != nil { + t.Fatalf("InitSnapshots: %v", err) + } + t.Cleanup(fsys.snapshotWkr.Stop) + return fsys +} + +func setxattr(t *testing.T, f *File, attr, value string) { + t.Helper() + if errno := f.Setxattr(context.Background(), attr, []byte(value), 0); errno != 0 { + t.Fatalf("setxattr %s=%q on inode %d: %v", attr, value, f.inodeNum, errno) + } +} + +func lookupFile(t *testing.T, fsys *FS, name string) *File { + t.Helper() + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + num, _, err := fsys.dirMgr.Lookup(context.Background(), root, name) + if err != nil { + t.Fatalf("lookup %s: %v", name, err) + } + return &File{fs: fsys, inodeNum: uint32(num)} +} + +func truncateFile(t *testing.T, f *File, size uint64) { + t.Helper() + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_SIZE + in.Size = size + if errno := f.Setattr(context.Background(), nil, in, &fuse.AttrOut{}); errno != 0 { + t.Fatalf("truncate inode %d to %d: %v", f.inodeNum, size, errno) + } +} + +// checkFile compares the whole file, including its size, with want. +func checkFile(t *testing.T, f *File, want []byte) { + t.Helper() + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.Size != uint64(len(want)) { + t.Fatalf("inode %d: size %d, want %d", f.inodeNum, ino.Size, len(want)) + } + checkAt(t, f, want, 0) +} + +func overwritten(base []byte, off int, data []byte) []byte { + out := bytes.Clone(base) + copy(out[off:], data) + return out +} + +func TestSnapshotKeepsDataThroughWrites(t *testing.T) { + path := newSnapshotFSImage(t) + fsys := openSnapshotFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatalf("InitRoot: %v", err) + } + original := pattern(1, 3*BlockSize) + + t.Run("SnapshotSurvivesWrites", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "before") + + writeAt(t, f, pattern(2, 4096), BlockSize+100) + writeAt(t, f, pattern(3, 2*BlockSize), 3*BlockSize) + truncateFile(t, f, BlockSize+BlockSize/2) + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + + live := overwritten(original[:BlockSize+BlockSize/2], BlockSize+100, pattern(2, 4096)) + for _, fsys := range []*FS{fsys, openSnapshotFS(t, path)} { + checkFile(t, lookupFile(t, fsys, "before-clone"), original) + checkFile(t, &File{fs: fsys, inodeNum: f.inodeNum}, live) + } + }) + + t.Run("CloneWritesStayInClone", func(t *testing.T) { + src := newRegularFile(t, fsys) + writeAt(t, src, original[:2*BlockSize], 0) + setxattr(t, src, "user.clone", "/clone") + clone := lookupFile(t, fsys, "clone") + + writeAt(t, clone, pattern(4, 100), 10) + writeAt(t, src, pattern(5, 100), BlockSize+10) + + for _, fsys := range []*FS{fsys, openSnapshotFS(t, path)} { + checkFile(t, &File{fs: fsys, inodeNum: src.inodeNum}, overwritten(original[:2*BlockSize], BlockSize+10, pattern(5, 100))) + checkFile(t, &File{fs: fsys, inodeNum: clone.inodeNum}, overwritten(original[:2*BlockSize], 10, pattern(4, 100))) + } + }) + + // Two blocks keep the copied block's split extents inline, so no tree node is allocated. + t.Run("DeleteAfterPartialOverwrite", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original[:2*BlockSize], 0) + free := fsys.bitmapMgr.FreeCount() + setxattr(t, f, "user.snapshot", "partial") + + writeAt(t, f, pattern(7, 4096), BlockSize+10) + if got := fsys.bitmapMgr.FreeCount(); got != free-1 { + t.Fatalf("free blocks after a write into 1 shared block = %d, want %d", got, free-1) + } + if errno := f.Removexattr(ctx, "user.snapshot.partial"); errno != 0 { + t.Fatalf("delete snapshot: %v", errno) + } + waitSnapshotGone(t, fsys, "partial") + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, f, overwritten(original[:2*BlockSize], BlockSize+10, pattern(7, 4096))) + }) + + t.Run("DeleteAfterOverwriteFreesSnapshotBlocks", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original[:2*BlockSize], 0) + free := fsys.bitmapMgr.FreeCount() + setxattr(t, f, "user.snapshot", "temp") + + writeAt(t, f, pattern(6, 2*BlockSize), 0) + if got := fsys.bitmapMgr.FreeCount(); got != free-2 { + t.Fatalf("free blocks after overwriting 2 shared blocks = %d, want %d", got, free-2) + } + + if errno := f.Removexattr(ctx, "user.snapshot.temp"); errno != 0 { + t.Fatalf("delete snapshot: %v", errno) + } + waitSnapshotGone(t, fsys, "temp") + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, f, pattern(6, 2*BlockSize)) + }) +} + +func waitSnapshotGone(t *testing.T, fsys *FS, name string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + _, err := fsys.snapshotReg.GetByName(context.Background(), name) + if errors.Is(err, snapshot.ErrSnapshotNotFound) { + return + } + if time.Now().After(deadline) { + t.Fatalf("snapshot %q still listed after 10s: %v", name, err) + } + time.Sleep(10 * time.Millisecond) + } +} + +func physicalBlock(t *testing.T, fsys *FS, num uint32, logical uint64) uint64 { + t.Helper() + ino, err := fsys.inodeTbl.Read(num) + if err != nil { + t.Fatal(err) + } + ext, off, err := fsys.extentMgr.Lookup(ino, logical) + if err != nil { + t.Fatal(err) + } + return ext.PhysicalBlock + off +} + +func refcountOf(t *testing.T, fsys *FS, block uint64) uint32 { + t.Helper() + e, err := fsys.refcountReg.Get(context.Background(), block) + if err != nil { + t.Fatal(err) + } + return e.RefCount +} + +// Older versions shared blocks without counting the owners, so this version +// refuses to share them again. +func TestSnapshotRefusesUncountedSharing(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + f := newRegularFile(t, fsys) + writeAt(t, f, pattern(1, BlockSize), 0) + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + ino.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + usedInodes := func() int { + n := 0 + for num := uint32(inode.InodeFirstUsable); num < 64; num++ { + if ino, err := fsys.inodeTbl.Read(num); err == nil && ino.Magic != 0 { + n++ + } + } + return n + } + used := usedInodes() + + if errno := f.Setxattr(ctx, "user.snapshot", []byte("legacy"), 0); errno != syscall.ENOTSUP { + t.Fatalf("snapshot of a file with uncounted sharing: errno %v, want ENOTSUP", errno) + } + if errno := f.Setxattr(ctx, "user.clone", []byte("/legacy-clone"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of a file with uncounted sharing: errno %v, want ENOTSUP", errno) + } + waitSnapshotGone(t, fsys, "legacy") + // The delete job frees the data inode right after removing the descriptor. + deadline := time.Now().Add(10 * time.Second) + for usedInodes() != used { + if time.Now().After(deadline) { + t.Fatalf("refused snapshot and clone left %d inodes in use, want %d", usedInodes(), used) + } + time.Sleep(10 * time.Millisecond) + } + checkFile(t, f, pattern(1, BlockSize)) +} + +// A new mount cleans up a snapshot a crash left half created or half deleted, +// and the file keeps its blocks. +func TestSnapshotRecoveryAfterCrash(t *testing.T) { + path := newSnapshotFSImage(t) + fsys := openSnapshotFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + content := pattern(3, 2*BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, content, 0) + blocks := []uint64{physicalBlock(t, fsys, f.inodeNum, 0), physicalBlock(t, fsys, f.inodeNum, 1)} + + // Crash after the blocks were shared, before the snapshot became active. + data, err := fsys.claimInode(ctx) + if err != nil { + t.Fatal(err) + } + data.Mode = inode.S_IFREG | 0o644 + data.Flags |= inode.FlagSnapshotData + if err := fsys.inodeTbl.Write(ctx, data); err != nil { + t.Fatal(err) + } + if _, err := fsys.snapshotReg.Create(ctx, "half-created", f.inodeNum, 0, data.InodeNum, 2, uint64(len(content)), snapshot.FlagReadOnly); err != nil { + t.Fatal(err) + } + if err := fsys.shareInto(ctx, f.inodeNum, data, nil); err != nil { + t.Fatal(err) + } + + // Crash after the data inode gave back its blocks, before the descriptor went away. + setxattr(t, f, "user.snapshot", "half-deleted") + deleting, err := fsys.snapshotReg.GetByName(ctx, "half-deleted") + if err != nil { + t.Fatal(err) + } + if err := fsys.snapshotReg.Delete(ctx, deleting.SnapshotID); err != nil { + t.Fatal(err) + } + if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.DataInode); err != nil { + t.Fatal(err) + } + + remounted := openSnapshotFS(t, path) + waitSnapshotGone(t, remounted, "half-created") + waitSnapshotGone(t, remounted, "half-deleted") + for _, num := range []uint32{data.InodeNum, deleting.DataInode} { + if ino, err := remounted.inodeTbl.Read(num); err != nil || ino.Magic != 0 { + t.Errorf("data inode %d still in use after recovery (err=%v)", num, err) + } + } + for _, b := range blocks { + if got := refcountOf(t, remounted, b); got > 1 { + t.Errorf("block %d refcount %d after recovery, want at most 1", b, got) + } + } + + // With no other owner left, a write goes in place. + g := &File{fs: remounted, inodeNum: f.inodeNum} + writeAt(t, g, pattern(4, 100), 10) + if got := physicalBlock(t, remounted, f.inodeNum, 0); got != blocks[0] { + t.Errorf("write after recovery moved block 0 from %d to %d", blocks[0], got) + } + checkFile(t, g, overwritten(content, 10, pattern(4, 100))) +} + +// Two nodes write into the same shared block at once. Node A commits its copy +// first; node B's commit then fails, B frees only its own copy and retries, +// and the old block loses exactly one owner. +func TestConcurrentCopyOnWriteAcrossNodes(t *testing.T) { + path := newSnapshotFSImage(t) + nodeA := openSnapshotFS(t, path) + if err := nodeA.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + original := pattern(1, BlockSize) + f := newRegularFile(t, nodeA) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "race") + old := physicalBlock(t, nodeA, f.inodeNum, 0) + + nodeB := openSnapshotFS(t, path) + t.Cleanup(func() { beforeCoWCommit = nil }) + beforeCoWCommit = func() { + beforeCoWCommit = nil + writeAt(t, f, pattern(2, 100), 0) + } + writeAt(t, &File{fs: nodeB, inodeNum: f.inodeNum}, pattern(3, 100), 200) + if beforeCoWCommit != nil { + t.Fatal("node B's write never reached a copy-on-write commit") + } + + after := openSnapshotFS(t, path) + if got := refcountOf(t, after, old); got != 1 { + t.Fatalf("old block refcount %d, want 1: the snapshot is its only owner", got) + } + current := physicalBlock(t, after, f.inodeNum, 0) + allocated := 0 + for b := uint64(0); b < snapFSDataBlocks; b++ { + if ok, err := after.bitmapMgr.IsAllocated(b); err != nil { + t.Fatal(err) + } else if ok { + allocated++ + } + } + // The root directory, the snapshot's old block and the file's copy; node B's copy is gone. + if allocated != 3 || current == old { + t.Fatalf("%d blocks allocated with the file on block %d (old %d); want 3 and a copy", allocated, current, old) + } + want := overwritten(overwritten(original, 0, pattern(2, 100)), 200, pattern(3, 100)) + checkFile(t, &File{fs: after, inodeNum: f.inodeNum}, want) + setxattr(t, &File{fs: after, inodeNum: f.inodeNum}, "user.snapshot.clone", "race:/race-clone") + checkFile(t, lookupFile(t, after, "race-clone"), original) +} + +// An image written by an older version, with an old snapshot and an old clone +// of one file. Writes, then deleting the clone and the source, never free a +// block another inode still maps; the blocks leak instead. +func TestOldSnapshotAndCloneNeverFreeSharedBlocks(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + src := newRegularFile(t, fsys) + writeAt(t, src, pattern(5, 2*BlockSize), 0) + + // What older versions left: blocks marked shared without FlagCounted, a + // refcount of 1 from the snapshot job, a clone of the same blocks whose + // refcount job never ran, and a snapshot descriptor without data. + srcIno, err := fsys.inodeTbl.Read(src.inodeNum) + if err != nil { + t.Fatal(err) + } + for i := range srcIno.InlineExtents { + if srcIno.InlineExtents[i].Length > 0 { + srcIno.InlineExtents[i].Flags |= extent.FlagShared + } + } + if err := fsys.inodeTbl.Write(ctx, srcIno); err != nil { + t.Fatal(err) + } + blocks, err := fsys.extentMgr.DataBlocks(srcIno) + if err != nil { + t.Fatal(err) + } + if _, err := fsys.refcountReg.IncrementBatch(ctx, blocks); err != nil { + t.Fatal(err) + } + clone := newRegularFile(t, fsys) + cloneIno, err := fsys.inodeTbl.Read(clone.inodeNum) + if err != nil { + t.Fatal(err) + } + cloneIno.InlineExtents = srcIno.InlineExtents + cloneIno.ExtentCount = srcIno.ExtentCount + cloneIno.Size, cloneIno.AllocatedBlocks, cloneIno.WrittenBlocks = srcIno.Size, srcIno.AllocatedBlocks, srcIno.WrittenBlocks + if err := fsys.inodeTbl.Write(ctx, cloneIno); err != nil { + t.Fatal(err) + } + old, err := fsys.snapshotReg.Create(ctx, "old-snap", src.inodeNum, 0, 0, 2, srcIno.Size, snapshot.FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := fsys.snapshotReg.Activate(ctx, old.SnapshotID); err != nil { + t.Fatal(err) + } + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + for name, num := range map[string]uint32{"src.bin": src.inodeNum, "clone.bin": clone.inodeNum} { + if err := fsys.dirMgr.Add(ctx, root, name, uint64(num), dir.DT_REG); err != nil { + t.Fatal(err) + } + } + stillAllocated := func(step string) { + t.Helper() + for _, b := range blocks { + if ok, err := fsys.bitmapMgr.IsAllocated(b); err != nil || !ok { + t.Fatalf("after %s, shared block %d is free (err=%v)", step, b, err) + } + } + } + + writeAt(t, clone, pattern(6, 100), 10) + stillAllocated("writing the old clone") + if errno := src.Setxattr(ctx, "user.snapshot.clone", []byte("old-snap:/x"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of a snapshot without data: errno %v, want ENOTSUP", errno) + } + if errno := src.Removexattr(ctx, "user.snapshot.old-snap"); errno != 0 { + t.Fatalf("delete old snapshot: %v", errno) + } + waitSnapshotGone(t, fsys, "old-snap") + stillAllocated("deleting the old snapshot") + if errno := fsys.Unlink(ctx, "clone.bin"); errno != 0 { + t.Fatalf("unlink old clone: %v", errno) + } + stillAllocated("deleting the old clone") + if errno := fsys.Unlink(ctx, "src.bin"); errno != 0 { + t.Fatalf("unlink source: %v", errno) + } + stillAllocated("deleting the source") +} + +// On an older version's image, file F had two snapshots (refcount 2) and then +// two clones whose owners were never counted. Copying on write must leave the +// old block to the other clone instead of releasing it. +func TestOldSharingCopiesWithoutReleasing(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(8, BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + fIno, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + fIno.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, fIno); err != nil { + t.Fatal(err) + } + old := physicalBlock(t, fsys, f.inodeNum, 0) + if err := fsys.refcountReg.Set(ctx, old, 2, refcount.FlagNone); err != nil { + t.Fatal(err) + } + var clones []*File + for range 2 { + c := newRegularFile(t, fsys) + cIno, err := fsys.inodeTbl.Read(c.inodeNum) + if err != nil { + t.Fatal(err) + } + cIno.InlineExtents, cIno.ExtentCount = fIno.InlineExtents, fIno.ExtentCount + cIno.Size, cIno.AllocatedBlocks, cIno.WrittenBlocks = fIno.Size, fIno.AllocatedBlocks, fIno.WrittenBlocks + if err := fsys.inodeTbl.Write(ctx, cIno); err != nil { + t.Fatal(err) + } + clones = append(clones, c) + } + + writeAt(t, f, pattern(9, 100), 10) + writeAt(t, clones[0], pattern(10, 100), 20) + + if ok, err := fsys.bitmapMgr.IsAllocated(old); err != nil || !ok { + t.Fatalf("old block %d freed while the second clone maps it (err=%v)", old, err) + } + if got := refcountOf(t, fsys, old); got != 2 { + t.Fatalf("old block refcount %d, want 2: copies of uncounted blocks must not release them", got) + } + checkFile(t, clones[1], original) + checkFile(t, f, overwritten(original, 10, pattern(9, 100))) + checkFile(t, clones[0], overwritten(original, 20, pattern(10, 100))) +} + +// A write over two shared blocks commits the first copy mid-write. When the +// final inode write then fails, each old block must lose this file's +// ownership exactly once across the failed write and its retry, and blocks an +// older version shared without counting must never lose any. +func TestFailedWriteReleasesCommittedCopiesOnce(t *testing.T) { + for _, legacy := range []bool{false, true} { + t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(1, 2*BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + old := []uint64{physicalBlock(t, fsys, f.inodeNum, 0), physicalBlock(t, fsys, f.inodeNum, 1)} + want := uint32(1) + if legacy { + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + ino.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + for _, b := range old { + if err := fsys.refcountReg.Set(ctx, b, 2, refcount.FlagNone); err != nil { + t.Fatal(err) + } + } + want = 2 + } else { + setxattr(t, f, "user.snapshot", "before") + } + + conflicts := 1 + testHookBeforeInodeCommit = func() error { + if conflicts > 0 { + conflicts-- + return extent.ErrTreeConflict + } + return nil + } + t.Cleanup(func() { testHookBeforeInodeCommit = nil }) + updated := pattern(2, 2*BlockSize) + writeAt(t, f, updated, 0) + if conflicts != 0 { + t.Fatal("the final inode write never ran") + } + + for _, b := range old { + if got := refcountOf(t, fsys, b); got != want { + t.Errorf("old block %d refcount %d, want %d", b, got, want) + } + if ok, err := fsys.bitmapMgr.IsAllocated(b); err != nil || !ok { + t.Errorf("old block %d freed (err=%v)", b, err) + } + } + checkFile(t, f, updated) + if !legacy { + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + checkFile(t, lookupFile(t, fsys, "before-clone"), original) + } + }) + } +} + +// An inode copy read before a copy-on-write committed, as by a slow chmod on +// another node, must not write the old mapping back: the old block already +// lost this file's ownership, so the snapshot would hold it at an undercount. +func TestStaleCopyCannotUndoCopyOnWrite(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(1, BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "before") + old := physicalBlock(t, fsys, f.inodeNum, 0) + + stale, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + writeAt(t, f, pattern(2, 100), 10) + stale.Mode = inode.S_IFREG | 0o600 + if err := fsys.inodeTbl.Write(ctx, stale); !errors.Is(err, inode.ErrTreeConflict) { + t.Fatalf("stale copy written over a committed copy-on-write: got %v, want ErrTreeConflict", err) + } + + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_MODE + in.Mode = 0o600 + if errno := f.Setattr(ctx, nil, in, &fuse.AttrOut{}); errno != 0 { + t.Fatalf("chmod: %v", errno) + } + if got := physicalBlock(t, fsys, f.inodeNum, 0); got == old { + t.Fatal("the file maps the snapshot's block again") + } + if got := refcountOf(t, fsys, old); got != 1 { + t.Fatalf("old block refcount %d, want 1", got) + } + checkFile(t, f, overwritten(original, 10, pattern(2, 100))) + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + checkFile(t, lookupFile(t, fsys, "before-clone"), original) +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 8daf0578..46924bba 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -64,6 +64,7 @@ const ( FlagSync uint16 = 1 << 3 // Synchronous updates FlagImmutable uint16 = 1 << 4 // Cannot be modified FlagAppend uint16 = 1 << 5 // Append-only + FlagSnapshotData uint16 = 1 << 6 // Holds a snapshot's frozen extents; in no directory ) // Errors @@ -273,6 +274,16 @@ func (t *Table) writeWithTree(inode *Inode) (func(), error) { return committed, nil } +// staleExtents reports whether writing inode would replace extents another +// write changed since inode was read, such as a copy-on-write whose old block +// was already released. +func staleExtents(inode, onDisk *Inode) bool { + if inode.HasExtentTree() || onDisk.HasExtentTree() { + return !sameExtentTree(inode, onDisk) + } + return onDisk.InlineExtents != inode.committedInline +} + func sameExtentTree(a, b *Inode) bool { return a.HasExtentTree() == b.HasExtentTree() && a.ExtentTreeRoot == b.ExtentTreeRoot && a.ExtentTreeGen == b.ExtentTreeGen && a.ExtentDepth == b.ExtentDepth @@ -506,7 +517,7 @@ func (t *Table) writeInodeSector(inode *Inode, checkTree bool) error { start := inodeOffset * InodeSize if checkTree { onDisk, err := deserializeInode(buf.Bytes()[start : start+InodeSize]) - if err == nil && (inode.HasExtentTree() || onDisk.HasExtentTree()) && !sameExtentTree(inode, onDisk) { + if err == nil && staleExtents(inode, onDisk) { return fmt.Errorf("%w: inode %d", ErrTreeConflict, inode.InodeNum) } } diff --git a/internal/inode/inode_test.go b/internal/inode/inode_test.go index 1124eecd..96f2dd20 100644 --- a/internal/inode/inode_test.go +++ b/internal/inode/inode_test.go @@ -985,3 +985,49 @@ func TestStaleCopyCannotReplaceExtentTree(t *testing.T) { t.Fatalf("a rejected write changed the inode: %+v", got) } } + +// A copy read before another write changed its inline extents, for example a +// copy-on-write that already released the old block, must not write the old +// mapping back. +func TestStaleCopyCannotUndoInlineExtentChange(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + var uuid [16]byte + tbl := NewTable(dev, 0, 16*InodeSize, nil, uuid) + ctx := context.Background() + + ino := &Inode{Magic: InodeMagic, InodeNum: 6, LinkCount: 1} + ino.InlineExtents[0] = InlineExtent{LogicalBlock: 0, PhysicalBlock: 9, Length: 1, Flags: 1} + if err := tbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + stale, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + copied, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + copied.InlineExtents[0].PhysicalBlock = 12 + if err := tbl.Write(ctx, copied); err != nil { + t.Fatal(err) + } + + stale.Mode = S_IFREG | 0o600 + if err := tbl.Write(ctx, stale); !errors.Is(err, ErrTreeConflict) { + t.Fatalf("stale inline copy: got %v, want ErrTreeConflict", err) + } + got, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + if got.InlineExtents[0].PhysicalBlock != 12 { + t.Fatalf("stale copy restored block %d", got.InlineExtents[0].PhysicalBlock) + } + // A copy that saw the change may write. + got.Mode = S_IFREG | 0o600 + if err := tbl.Write(ctx, got); err != nil { + t.Fatalf("fresh copy: %v", err) + } +} diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 6863179c..d8682f4b 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -1,11 +1,13 @@ // Package refcount implements block reference counting for GoClusterFS snapshots. // // Reference counting enables Copy-on-Write (CoW) semantics for snapshots: -// - RefCount = 1: Block is exclusive to one file, can modify in place -// - RefCount > 1: Block is shared (via snapshot), must CoW before writing +// - RefCount = 0 or 1: Block has one owner and can be modified in place. +// New blocks start at 0, so allocation never touches this region. +// - RefCount > 1: Block is shared by that many owners (files, clones, +// snapshots) and must be copied before writing. // -// This allows O(1) snapshot creation - just increment refcounts of all blocks. -// When a write targets a shared block, allocate a new block and copy data. +// Share adds an owner and Release drops one; a block goes back to the bitmap +// only when Release reports that its last owner let go. // // Layout: Each entry is 8 bytes (4-byte count + 2-byte flags + 2-byte reserved). // Entries are stored in 4KB sectors with a CRC32C checksum at the end. @@ -272,82 +274,91 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags // This is more efficient than calling Increment individually for each block. // Returns the new counts after incrementing. func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { - if len(blockNums) == 0 { - return nil, nil - } - - for _, blockNum := range blockNums { - if blockNum >= r.totalBlocks { - return nil, ErrBlockOutOfRange + results := make([]uint32, len(blockNums)) + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount == 0xFFFFFFFF { + return false, ErrRefCountOverflow } + e.RefCount++ + results[i] = e.RefCount + return true, nil + }) + if err != nil || len(blockNums) == 0 { + return nil, err } + return results, nil +} - // Acquire distributed lock if available - if r.lockMgr != nil { - if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire refcount lock: %w", err) +// DecrementBatch decrements reference counts for multiple blocks atomically. +// Returns the new counts after decrementing. +// Returns ErrRefCountUnderflow if any block's count is already zero. +func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { + results := make([]uint32, len(blockNums)) + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount == 0 { + return false, ErrRefCountUnderflow } - defer r.lockMgr.Release(lock.LockTypeRefCount, 0) + e.RefCount-- + results[i] = e.RefCount + return true, nil + }) + if err != nil || len(blockNums) == 0 { + return nil, err } + return results, nil +} - r.mu.Lock() - defer r.mu.Unlock() - - results := make([]uint32, len(blockNums)) - - // Group blocks by sector for efficient I/O - sectorCache := make(map[uint64][]byte) - - for i, blockNum := range blockNums { - sectorIdx := blockNum / EntriesPerSector - - // Load sector if not cached - if _, ok := sectorCache[sectorIdx]; !ok { - sectorData, err := r.readSector(sectorIdx) - if err != nil { - return nil, err - } - sectorCache[sectorIdx] = sectorData +// Share adds one owner, such as a snapshot or a clone, to each block. Blocks +// are allocated with a count of 0, which like 1 means a single owner, so the +// first share of a block sets its count to 2. +func (r *Region) Share(ctx context.Context, blockNums []uint64) error { + return r.updateBatch(ctx, blockNums, func(_ int, e *Entry) (bool, error) { + if e.RefCount == 0xFFFFFFFF { + return false, ErrRefCountOverflow } + e.RefCount = max(e.RefCount, 1) + 1 + return true, nil + }) +} - // Read and increment entry - entry := decodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector) - if entry.RefCount == 0xFFFFFFFF { - return nil, ErrRefCountOverflow +// Release drops one owner from each block and returns the blocks that no +// owner holds any more. The caller must free those in the bitmap. +func (r *Region) Release(ctx context.Context, blockNums []uint64) ([]uint64, error) { + var free []uint64 + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount > 1 { + e.RefCount-- + return true, nil } - entry.RefCount++ - results[i] = entry.RefCount - encodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector, entry) - } - - // Write back modified sectors - for sectorIdx, sectorData := range sectorCache { - if err := r.writeSector(sectorIdx, sectorData); err != nil { - return nil, err + free = append(free, blockNums[i]) + if *e == (Entry{}) { + return false, nil } + *e = Entry{} + return true, nil + }) + if err != nil { + return nil, err } - - return results, nil + return free, nil } -// DecrementBatch decrements reference counts for multiple blocks atomically. -// Returns the new counts after decrementing. -// Returns ErrRefCountUnderflow if any block's count is already zero. -func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { +// updateBatch applies fn to the entry of each block under the region lock and +// writes back the sectors fn changed. Nothing is written if fn fails. +func (r *Region) updateBatch(ctx context.Context, blockNums []uint64, fn func(i int, e *Entry) (bool, error)) error { if len(blockNums) == 0 { - return nil, nil + return nil } - for _, blockNum := range blockNums { if blockNum >= r.totalBlocks { - return nil, ErrBlockOutOfRange + return ErrBlockOutOfRange } } // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire refcount lock: %w", err) + return fmt.Errorf("failed to acquire refcount lock: %w", err) } defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } @@ -355,41 +366,34 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint r.mu.Lock() defer r.mu.Unlock() - results := make([]uint32, len(blockNums)) - - // Group blocks by sector for efficient I/O - sectorCache := make(map[uint64][]byte) - + sectors := make(map[uint64][]byte) + dirty := make(map[uint64]bool) for i, blockNum := range blockNums { sectorIdx := blockNum / EntriesPerSector - - // Load sector if not cached - if _, ok := sectorCache[sectorIdx]; !ok { - sectorData, err := r.readSector(sectorIdx) + if _, ok := sectors[sectorIdx]; !ok { + data, err := r.readSector(sectorIdx) if err != nil { - return nil, err + return err } - sectorCache[sectorIdx] = sectorData + sectors[sectorIdx] = data } - - // Read and decrement entry - entry := decodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector) - if entry.RefCount == 0 { - return nil, ErrRefCountUnderflow + entry := decodeEntry(sectors[sectorIdx], blockNum%EntriesPerSector) + changed, err := fn(i, &entry) + if err != nil { + return err + } + if changed { + encodeEntry(sectors[sectorIdx], blockNum%EntriesPerSector, entry) + dirty[sectorIdx] = true } - entry.RefCount-- - results[i] = entry.RefCount - encodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector, entry) } - // Write back modified sectors - for sectorIdx, sectorData := range sectorCache { - if err := r.writeSector(sectorIdx, sectorData); err != nil { - return nil, err + for sectorIdx := range dirty { + if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil { + return err } } - - return results, nil + return nil } // Scan reads all reference counts and returns statistics. diff --git a/internal/refcount/refcount_test.go b/internal/refcount/refcount_test.go index beb18bc8..a05f8123 100644 --- a/internal/refcount/refcount_test.go +++ b/internal/refcount/refcount_test.go @@ -566,6 +566,61 @@ func TestDecrementBatchUnderflow(t *testing.T) { } } +func TestShareAndRelease(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + region := NewRegion(dev, 0, 1000, nil) + ctx := context.Background() + blocks := []uint64{3, 700} + count := func(block uint64) uint32 { + t.Helper() + e, err := region.Get(ctx, block) + if err != nil { + t.Fatal(err) + } + return e.RefCount + } + + if err := region.Share(ctx, blocks); err != nil { + t.Fatal(err) + } + if got := count(3); got != 2 { + t.Fatalf("first share of a new block: count %d, want 2", got) + } + if err := region.Share(ctx, blocks[:1]); err != nil { + t.Fatal(err) + } + if got := count(3); got != 3 { + t.Fatalf("second share: count %d, want 3", got) + } + + for want := uint32(2); want >= 1; want-- { + free, err := region.Release(ctx, blocks[:1]) + if err != nil { + t.Fatal(err) + } + if len(free) != 0 || count(3) != want { + t.Fatalf("release down to %d: free=%v count=%d", want, free, count(3)) + } + } + free, err := region.Release(ctx, blocks) + if err != nil { + t.Fatal(err) + } + if len(free) != 1 || free[0] != 3 || count(3) != 0 || count(700) != 1 { + t.Fatalf("last owner of 3 and one of two owners of 700: free=%v counts=%d,%d", free, count(3), count(700)) + } + + free, err = region.Release(ctx, []uint64{700, 900}) + if err != nil { + t.Fatal(err) + } + if len(free) != 2 || count(700) != 0 || count(900) != 0 { + t.Fatalf("release of single-owner blocks: free=%v", free) + } +} + func TestScan(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index ddc0f9a8..dca8b8f6 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -21,6 +21,7 @@ import ( "time" "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -105,6 +106,11 @@ const ( // FlagAutoDelete indicates the snapshot should be auto-deleted // when no longer referenced. FlagAutoDelete Flags = 1 << 3 + + // FlagHasData marks the current descriptor format: DataInode names a hidden + // inode that maps the blocks the snapshot captured. Descriptors written by + // older versions lack it and hold no data. + FlagHasData Flags = 1 << 4 ) // Errors @@ -135,7 +141,8 @@ var ( // 24 1 State (Creating/Active/Deleting) // 25 1 Flags (ReadOnly/Writable/Hidden) // 26 2 Reserved1 -// 28 8 ExtentTreeRoot (copied from inode at snapshot time) +// 28 4 DataInode (hidden inode with the captured extents; only with FlagHasData) +// 32 4 Reserved2 (older versions wrote an unused extent tree root at 28-35) // 36 8 TotalBlocks (block count at snapshot time) // 44 8 TotalSize (file size at snapshot time) // 52 8 CreateTime (Unix timestamp) @@ -144,21 +151,22 @@ var ( // 76 48 Name (null-terminated UTF-8 string) // 124 4 Checksum (CRC32C of bytes 0-123) type Descriptor struct { - Magic uint32 // 0x534E4150 "SNAP" - SnapshotID uint64 // Unique snapshot identifier - ParentID uint64 // Parent snapshot (0 for root) - TargetInode uint32 // Inode number of snapshotted file - State State // Current state - Flags Flags // Snapshot flags - Reserved1 uint16 // Alignment padding - ExtentTreeRoot uint64 // Extent tree root block - TotalBlocks uint64 // Number of blocks at snapshot time - TotalSize uint64 // File size at snapshot time - CreateTime int64 // Creation timestamp - DeleteTime int64 // Deletion timestamp (0 if active) - RefCountJobID uint64 // Background refcount job ID - Name [MaxSnapshotNameLen]byte // Snapshot name - Checksum uint32 // CRC32C checksum + Magic uint32 // 0x534E4150 "SNAP" + SnapshotID uint64 // Unique snapshot identifier + ParentID uint64 // Parent snapshot (0 for root) + TargetInode uint32 // Inode number of snapshotted file + State State // Current state + Flags Flags // Snapshot flags + Reserved1 uint16 // Alignment padding + DataInode uint32 // Hidden inode holding the captured extents + Reserved2 uint32 // Reserved + TotalBlocks uint64 // Number of blocks at snapshot time + TotalSize uint64 // File size at snapshot time + CreateTime int64 // Creation timestamp + DeleteTime int64 // Deletion timestamp (0 if active) + RefCountJobID uint64 // Background refcount job ID + Name [MaxSnapshotNameLen]byte // Snapshot name + Checksum uint32 // CRC32C checksum } // IsValid returns true if this is a valid snapshot descriptor. @@ -166,6 +174,11 @@ func (d *Descriptor) IsValid() bool { return d.Magic == DescriptorMagic && d.State != StateInvalid } +// HasData reports whether the snapshot captured data that can be cloned. +func (d *Descriptor) HasData() bool { + return d.Flags&FlagHasData != 0 && d.DataInode != 0 +} + // IsFree returns true if this slot is free (can be reused). func (d *Descriptor) IsFree() bool { return d.Magic == 0 || d.State == StateInvalid @@ -218,10 +231,21 @@ type Registry struct { size uint64 // Region size in bytes maxDescriptors uint64 // Maximum number of descriptors lockMgr *lock.Manager // For distributed locking (optional) + nodeSlot int // This node's slot in the node registry nextID uint64 // Next available snapshot ID (cached) descriptorCount uint64 // Active descriptor count (cached) } +// SetNodeSlot tells the registry which node slot this mount holds. New IDs +// end in the slot and each node fills its own sectors first, so two nodes +// never create the same ID or write the same sector at once, even if the +// registry lock fails to exclude them. +func (r *Registry) SetNodeSlot(slot int) { + r.mu.Lock() + defer r.mu.Unlock() + r.nodeSlot = slot % node.MaxNodes +} + // NewRegistry creates a new snapshot registry manager. // // Parameters: @@ -247,14 +271,14 @@ func NewRegistry(dev *storage.Device, offset, size uint64, lockMgr *lock.Manager // - name: Human-readable snapshot name // - targetInode: Inode number being snapshotted // - parentID: Parent snapshot ID (0 for root snapshot) -// - extentTreeRoot: Extent tree root block from the inode +// - dataInode: Hidden inode holding the captured extents (0 for none) // - totalBlocks: Number of blocks at snapshot time // - totalSize: File size at snapshot time // - flags: Snapshot flags (ReadOnly, Writable, etc.) // // Returns the new snapshot ID and descriptor. func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, - parentID uint64, extentTreeRoot uint64, totalBlocks uint64, + parentID uint64, dataInode uint32, totalBlocks uint64, totalSize uint64, flags Flags) (*Descriptor, error) { if len(name) >= MaxSnapshotNameLen { @@ -273,28 +297,28 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, r.mu.Lock() defer r.mu.Unlock() - // Find a free slot - slot, err := r.findFreeSlot() + slot, maxID, err := r.freeSlotAndMaxID() if err != nil { return nil, err } + base := max(maxID, r.nextID-1) + snapshotID := (base/node.MaxNodes+1)*node.MaxNodes + uint64(r.nodeSlot) + r.nextID = snapshotID + 1 - // Allocate a new ID - snapshotID := r.nextID - r.nextID++ - - // Create the descriptor + if dataInode != 0 { + flags |= FlagHasData + } desc := &Descriptor{ - Magic: DescriptorMagic, - SnapshotID: snapshotID, - ParentID: parentID, - TargetInode: targetInode, - State: StateCreating, // Will be marked Active after refcount job - Flags: flags | FlagReadOnly, - ExtentTreeRoot: extentTreeRoot, - TotalBlocks: totalBlocks, - TotalSize: totalSize, - CreateTime: time.Now().Unix(), + Magic: DescriptorMagic, + SnapshotID: snapshotID, + ParentID: parentID, + TargetInode: targetInode, + State: StateCreating, // Will be marked Active after refcount job + Flags: flags | FlagReadOnly, + DataInode: dataInode, + TotalBlocks: totalBlocks, + TotalSize: totalSize, + CreateTime: time.Now().Unix(), } if err := desc.SetName(name); err != nil { return nil, err @@ -447,6 +471,25 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u // UpdateState updates the state of a snapshot. func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState State) error { + return r.updateState(ctx, snapshotID, newState, StateInvalid) +} + +// Activate marks a snapshot that is being created as active. It fails with +// ErrSnapshotInvalidState if the snapshot left the creating state meanwhile, +// for example because another node's mount rolled it back. +func (r *Registry) Activate(ctx context.Context, snapshotID uint64) error { + return r.updateState(ctx, snapshotID, StateActive, StateCreating) +} + +// Abandon marks a snapshot that is still being created as deleting. It fails +// with ErrSnapshotInvalidState if the snapshot became active meanwhile. +func (r *Registry) Abandon(ctx context.Context, snapshotID uint64) error { + return r.updateState(ctx, snapshotID, StateDeleting, StateCreating) +} + +// updateState sets a snapshot's state, requiring it to be in from unless from +// is StateInvalid. +func (r *Registry) updateState(ctx context.Context, snapshotID uint64, newState, from State) error { if snapshotID == InvalidSnapshotID { return ErrInvalidID } @@ -469,6 +512,9 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState continue } if desc.IsValid() && desc.SnapshotID == snapshotID { + if from != StateInvalid && desc.State != from { + return ErrSnapshotInvalidState + } desc.State = newState if newState == StateDeleting { desc.DeleteTime = time.Now().Unix() @@ -647,19 +693,38 @@ type ScanStats struct { TotalBlocks uint64 // Total blocks referenced by all snapshots } -// findFreeSlot finds the first free slot in the registry. -func (r *Registry) findFreeSlot() (uint64, error) { - for slot := uint64(0); slot < r.maxDescriptors; slot++ { - desc, err := r.readDescriptor(slot) - if err != nil { - // Treat read errors as potentially free (might be uninitialized) - return slot, nil - } - if desc.IsFree() { - return slot, nil +// freeSlotAndMaxID returns a free slot, preferring sectors this node owns, and +// the highest snapshot ID on disk, which includes snapshots other nodes +// created since this node's scan. Unreadable slots count as free, as they did +// before. +func (r *Registry) freeSlotAndMaxID() (uint64, uint64, error) { + free, owned := r.maxDescriptors, r.maxDescriptors + var maxID uint64 + for sector := uint64(0); sector*DescriptorsPerSector < r.maxDescriptors; sector++ { + data, readErr := r.readSector(sector) + for i := uint64(0); i < DescriptorsPerSector; i++ { + slot := sector*DescriptorsPerSector + i + if slot >= r.maxDescriptors { + break + } + desc, err := decodeDescriptor(data, i) + if readErr != nil || err != nil || desc.IsFree() { + free = min(free, slot) + if sector%node.MaxNodes == uint64(r.nodeSlot) { + owned = min(owned, slot) + } + continue + } + maxID = max(maxID, desc.SnapshotID) } } - return 0, ErrSnapshotFull + if owned < r.maxDescriptors { + return owned, maxID, nil + } + if free == r.maxDescriptors { + return 0, 0, ErrSnapshotFull + } + return free, maxID, nil } // readDescriptor reads a descriptor from a slot. @@ -787,20 +852,21 @@ func decodeDescriptor(sectorData []byte, descIdx uint64) (*Descriptor, error) { } desc := &Descriptor{ - Magic: magic, - SnapshotID: binary.LittleEndian.Uint64(data[4:12]), - ParentID: binary.LittleEndian.Uint64(data[12:20]), - TargetInode: binary.LittleEndian.Uint32(data[20:24]), - State: State(data[24]), - Flags: Flags(data[25]), - Reserved1: binary.LittleEndian.Uint16(data[26:28]), - ExtentTreeRoot: binary.LittleEndian.Uint64(data[28:36]), - TotalBlocks: binary.LittleEndian.Uint64(data[36:44]), - TotalSize: binary.LittleEndian.Uint64(data[44:52]), - CreateTime: int64(binary.LittleEndian.Uint64(data[52:60])), - DeleteTime: int64(binary.LittleEndian.Uint64(data[60:68])), - RefCountJobID: binary.LittleEndian.Uint64(data[68:76]), - Checksum: stored, + Magic: magic, + SnapshotID: binary.LittleEndian.Uint64(data[4:12]), + ParentID: binary.LittleEndian.Uint64(data[12:20]), + TargetInode: binary.LittleEndian.Uint32(data[20:24]), + State: State(data[24]), + Flags: Flags(data[25]), + Reserved1: binary.LittleEndian.Uint16(data[26:28]), + DataInode: binary.LittleEndian.Uint32(data[28:32]), + Reserved2: binary.LittleEndian.Uint32(data[32:36]), + TotalBlocks: binary.LittleEndian.Uint64(data[36:44]), + TotalSize: binary.LittleEndian.Uint64(data[44:52]), + CreateTime: int64(binary.LittleEndian.Uint64(data[52:60])), + DeleteTime: int64(binary.LittleEndian.Uint64(data[60:68])), + RefCountJobID: binary.LittleEndian.Uint64(data[68:76]), + Checksum: stored, } copy(desc.Name[:], data[76:124]) @@ -829,7 +895,8 @@ func encodeDescriptor(sectorData []byte, descIdx uint64, desc *Descriptor) { data[24] = byte(desc.State) data[25] = byte(desc.Flags) binary.LittleEndian.PutUint16(data[26:28], desc.Reserved1) - binary.LittleEndian.PutUint64(data[28:36], desc.ExtentTreeRoot) + binary.LittleEndian.PutUint32(data[28:32], desc.DataInode) + binary.LittleEndian.PutUint32(data[32:36], desc.Reserved2) binary.LittleEndian.PutUint64(data[36:44], desc.TotalBlocks) binary.LittleEndian.PutUint64(data[44:52], desc.TotalSize) binary.LittleEndian.PutUint64(data[52:60], uint64(desc.CreateTime)) diff --git a/internal/snapshot/snapshot_fuzz_test.go b/internal/snapshot/snapshot_fuzz_test.go index f16c4c81..3b167ffe 100644 --- a/internal/snapshot/snapshot_fuzz_test.go +++ b/internal/snapshot/snapshot_fuzz_test.go @@ -10,17 +10,17 @@ import ( func FuzzDecodeDescriptor(f *testing.F) { active := Descriptor{ - Magic: DescriptorMagic, - SnapshotID: 7, - ParentID: 3, - TargetInode: 42, - State: StateActive, - Flags: FlagReadOnly, - ExtentTreeRoot: 1234, - TotalBlocks: 5, - TotalSize: 5 << 20, - CreateTime: 1_700_000_000, - RefCountJobID: 9, + Magic: DescriptorMagic, + SnapshotID: 7, + ParentID: 3, + TargetInode: 42, + State: StateActive, + Flags: FlagReadOnly | FlagHasData, + DataInode: 1234, + TotalBlocks: 5, + TotalSize: 5 << 20, + CreateTime: 1_700_000_000, + RefCountJobID: 9, } if err := active.SetName("nightly"); err != nil { f.Fatal(err) diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index f38e9ec7..deb8903f 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -2,9 +2,11 @@ package snapshot import ( "context" + "errors" "fmt" "os" "path/filepath" + "sync" "testing" "github.com/supporttools/GoClusterFS/internal/storage" @@ -257,8 +259,8 @@ func TestRegistryCreate(t *testing.T) { if desc.GetName() != "snap1" { t.Errorf("Name = %q, want %q", desc.GetName(), "snap1") } - if desc.ExtentTreeRoot != 1000 { - t.Errorf("ExtentTreeRoot = %d, want 1000", desc.ExtentTreeRoot) + if desc.DataInode != 1000 { + t.Errorf("DataInode = %d, want 1000", desc.DataInode) } if desc.TotalBlocks != 50 { t.Errorf("TotalBlocks = %d, want 50", desc.TotalBlocks) @@ -681,8 +683,8 @@ func TestRegistryPersistence(t *testing.T) { if desc.State != StateActive { t.Errorf("State = %v, want %v", desc.State, StateActive) } - if desc.ExtentTreeRoot != 1000 { - t.Errorf("ExtentTreeRoot = %d, want 1000", desc.ExtentTreeRoot) + if desc.DataInode != 1000 { + t.Errorf("DataInode = %d, want 1000", desc.DataInode) } } @@ -740,7 +742,7 @@ func TestMultipleSectors(t *testing.T) { for i := 0; i < 50; i++ { name := fmt.Sprintf("snap%d", i) _, err := registry.Create(ctx, name, uint32(100+i), InvalidSnapshotID, - uint64(1000+i), uint64(50+i), uint64(1024*(i+1)), FlagReadOnly) + uint32(1000+i), uint64(50+i), uint64(1024*(i+1)), FlagReadOnly) if err != nil { t.Fatalf("Create(%s) failed: %v", name, err) } @@ -850,3 +852,180 @@ func BenchmarkList(b *testing.B) { registry.List(ctx) } } + +// Two mounted nodes each create a snapshot; the IDs must differ because a +// delete by ID would otherwise remove the other node's snapshot. +func TestRegistryIDsUniqueAcrossNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + ctx := context.Background() + + nodeA := NewRegistry(dev, 0, 10*SectorSize, nil) + nodeB := NewRegistry(dev, 0, 10*SectorSize, nil) + for slot, r := range []*Registry{nodeA, nodeB} { + r.SetNodeSlot(slot) + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + } + a, err := nodeA.Create(ctx, "from-a", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + b, err := nodeB.Create(ctx, "from-b", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if a.SnapshotID == b.SnapshotID { + t.Fatalf("both nodes created snapshot ID %d", a.SnapshotID) + } +} + +func TestRegistryActivateRequiresCreating(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + registry := NewRegistry(dev, 0, 10*SectorSize, nil) + ctx := context.Background() + + desc, err := registry.Create(ctx, "rolled-back", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := registry.Delete(ctx, desc.SnapshotID); err != nil { + t.Fatal(err) + } + if err := registry.Activate(ctx, desc.SnapshotID); !errors.Is(err, ErrSnapshotInvalidState) { + t.Fatalf("Activate of a deleting snapshot: got %v, want ErrSnapshotInvalidState", err) + } + if got, err := registry.Get(ctx, desc.SnapshotID); err != nil || got.State != StateDeleting { + t.Fatalf("snapshot after refused Activate: %+v, %v; want deleting", got, err) + } + + fresh, err := registry.Create(ctx, "fresh", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := registry.Activate(ctx, fresh.SnapshotID); err != nil { + t.Fatalf("Activate of a creating snapshot: %v", err) + } + // Another node's mount must not roll back a snapshot that became active. + if err := registry.Abandon(ctx, fresh.SnapshotID); !errors.Is(err, ErrSnapshotInvalidState) { + t.Fatalf("Abandon of an active snapshot: got %v, want ErrSnapshotInvalidState", err) + } + if got, err := registry.Get(ctx, fresh.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("snapshot after refused Abandon: %+v, %v; want active", got, err) + } +} + +// Two nodes create snapshots at the same time with no lock between them, as +// when the distributed lock fails to exclude them. Node slots alone must keep +// every snapshot in its own slot with its own ID. +func TestRegistryConcurrentCreatesAcrossNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 4<<20) + defer cleanup() + ctx := context.Background() + const perNode = 20 + + var nodes []*Registry + for slot := range 2 { + r := NewRegistry(dev, 0, 10*SectorSize, nil) + r.SetNodeSlot(slot) + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + nodes = append(nodes, r) + } + + var wg sync.WaitGroup + errs := make(chan error, 2*perNode) + for n, r := range nodes { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perNode { + if _, err := r.Create(ctx, fmt.Sprintf("node%d-%d", n, i), 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly); err != nil { + errs <- err + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + all, err := nodes[0].List(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 2*perNode { + t.Fatalf("%d snapshots on disk, want %d", len(all), 2*perNode) + } + names := make(map[uint64]string) + for _, d := range all { + if other, dup := names[d.SnapshotID]; dup { + t.Fatalf("snapshots %q and %q share ID %d", other, d.GetName(), d.SnapshotID) + } + names[d.SnapshotID] = d.GetName() + } +} + +// Older versions numbered snapshots 1, 2, 3... New IDs must land above every +// one of them, and a node that takes over a departed node's slot must never +// reuse that node's IDs. +func TestRegistryIDsAfterOlderImagesAndSlotReuse(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + ctx := context.Background() + + old := NewRegistry(dev, 0, 10*SectorSize, nil) + for i := range 70 { + if _, err := old.Create(ctx, fmt.Sprintf("old-%d", i), 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly); err != nil { + t.Fatal(err) + } + } + // Rewrite them with the sequential IDs an older version would have assigned. + all, err := old.List(ctx) + if err != nil { + t.Fatal(err) + } + for slot := uint64(0); slot < old.maxDescriptors; slot++ { + d, err := old.readDescriptor(slot) + if err != nil || d.IsFree() { + continue + } + var n int + fmt.Sscanf(d.GetName(), "old-%d", &n) + d.SnapshotID = uint64(n + 1) + if err := old.writeDescriptor(slot, d); err != nil { + t.Fatal(err) + } + } + if len(all) != 70 { + t.Fatalf("%d old snapshots, want 70", len(all)) + } + + create := func(slot int, name string) uint64 { + t.Helper() + r := NewRegistry(dev, 0, 10*SectorSize, nil) + r.SetNodeSlot(slot) + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + d, err := r.Create(ctx, name, 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + return d.SnapshotID + } + first := create(5, "from-slot-5") + if first <= 70 || first%64 != 5 { + t.Fatalf("ID %d after IDs 1-70 from an older image, want above 70 and ending in slot 5", first) + } + // A new node takes over slot 5 after the first owner left. + second := create(5, "slot-5-new-owner") + if second <= first || second%64 != 5 { + t.Fatalf("new owner of slot 5 got ID %d after %d, want a higher ID ending in slot 5", second, first) + } +} diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index 63b0652e..ec4b7b26 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -1,44 +1,30 @@ // Package snapshot implements snapshot management for GoClusterFS. // -// worker.go implements the background worker for asynchronous refcount updates. -// When a snapshot is created, the descriptor is written immediately (O(1)), -// and the refcount increments are queued for background processing. +// worker.go implements the background worker that deletes snapshots and +// cleans up snapshots a crash left half created or half deleted. package snapshot import ( "context" + "errors" "fmt" "log" "sync" "sync/atomic" "time" - "github.com/supporttools/GoClusterFS/internal/bitmap" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" - "github.com/supporttools/GoClusterFS/internal/refcount" ) -// JobType represents the type of background job. -type JobType uint8 +// ErrWorkerStopped is returned when a job is queued on a stopped worker. +var ErrWorkerStopped = errors.New("snapshot worker stopped") -const ( - // JobTypeCreate processes refcount increments for snapshot creation. - JobTypeCreate JobType = 1 - - // JobTypeDelete processes refcount decrements for snapshot deletion. - JobTypeDelete JobType = 2 -) - -// Job represents a background refcount job. +// Job is a queued snapshot deletion. type Job struct { ID uint64 // Unique job identifier - Type JobType // Create or Delete - SnapshotID uint64 // Snapshot being processed - InodeNum uint32 // Inode number of the source file - BlockNums []uint64 // Block numbers to process + SnapshotID uint64 // Snapshot being deleted StartTime time.Time // When job was queued - Retries int // Number of retry attempts } // JobResult contains the result of a completed job. @@ -50,14 +36,12 @@ type JobResult struct { Duration time.Duration } -// Worker processes background refcount jobs for snapshots. +// Worker deletes snapshots in the background. type Worker struct { mu sync.RWMutex registry *Registry - refRegion *refcount.Region extentMgr *extent.Manager inodeTable *inode.Table - bitmapMgr *bitmap.Region // For freeing blocks when refcount reaches 0 jobQueue chan *Job resultChan chan *JobResult @@ -91,17 +75,9 @@ func DefaultWorkerConfig() WorkerConfig { } } -// NewWorker creates a new background worker. -// Note: bitmapMgr can be nil if block freeing is handled elsewhere, but for -// proper snapshot deletion with block reclamation, it should be provided. -func NewWorker(registry *Registry, refRegion *refcount.Region, extentMgr *extent.Manager, inodeTable *inode.Table, cfg WorkerConfig) *Worker { - return NewWorkerWithBitmap(registry, refRegion, extentMgr, inodeTable, nil, cfg) -} - -// NewWorkerWithBitmap creates a new background worker with bitmap for block freeing. -// When a block's refcount reaches 0 during snapshot deletion, the bitmap is used -// to mark the block as free for reallocation. -func NewWorkerWithBitmap(registry *Registry, refRegion *refcount.Region, extentMgr *extent.Manager, inodeTable *inode.Table, bitmapMgr *bitmap.Region, cfg WorkerConfig) *Worker { +// NewWorker creates a new background worker. Blocks released by a deleted +// snapshot are freed through extentMgr, which must have refcounts set. +func NewWorker(registry *Registry, extentMgr *extent.Manager, inodeTable *inode.Table, cfg WorkerConfig) *Worker { if cfg.QueueSize <= 0 { cfg.QueueSize = 100 } @@ -111,10 +87,8 @@ func NewWorkerWithBitmap(registry *Registry, refRegion *refcount.Region, extentM w := &Worker{ registry: registry, - refRegion: refRegion, extentMgr: extentMgr, inodeTable: inodeTable, - bitmapMgr: bitmapMgr, jobQueue: make(chan *Job, cfg.QueueSize), resultChan: make(chan *JobResult, cfg.QueueSize), stopChan: make(chan struct{}), @@ -155,72 +129,32 @@ func (w *Worker) Stop() { log.Printf("[snapshot/worker] Stopped") } -// QueueCreateJob queues a refcount increment job for snapshot creation. -// Returns the job ID immediately. -func (w *Worker) QueueCreateJob(ctx context.Context, snapshotID uint64, inodeNum uint32, blockNums []uint64) (uint64, error) { - // The snapshot descriptor already exists, so the job must be queued even if the caller gives up. - ctx = context.WithoutCancel(ctx) - +// QueueDeleteJob queues the deletion of a snapshot that is already marked +// deleting. The job outlives the request that queued it, so ctx only carries +// values; a full queue makes the caller wait. +func (w *Worker) QueueDeleteJob(ctx context.Context, snapshotID uint64) (uint64, error) { if !w.running.Load() { - return 0, fmt.Errorf("worker not running") - } - - jobID := w.nextJobID.Add(1) - 1 - job := &Job{ - ID: jobID, - Type: JobTypeCreate, - SnapshotID: snapshotID, - InodeNum: inodeNum, - BlockNums: blockNums, - StartTime: time.Now(), - } - - // Store job ID in snapshot descriptor - if err := w.registry.SetRefCountJobID(ctx, snapshotID, jobID); err != nil { - return 0, fmt.Errorf("failed to set job ID: %w", err) - } - - select { - case w.jobQueue <- job: - log.Printf("[snapshot/worker] Queued create job %d for snapshot %d (%d blocks)", - jobID, snapshotID, len(blockNums)) - return jobID, nil - default: - return 0, fmt.Errorf("job queue full") + return 0, ErrWorkerStopped } -} - -// QueueDeleteJob queues a refcount decrement job for snapshot deletion. -// Returns the job ID immediately. -func (w *Worker) QueueDeleteJob(ctx context.Context, snapshotID uint64, blockNums []uint64) (uint64, error) { - // The snapshot descriptor already exists, so the job must be queued even if the caller gives up. ctx = context.WithoutCancel(ctx) - if !w.running.Load() { - return 0, fmt.Errorf("worker not running") - } - jobID := w.nextJobID.Add(1) - 1 job := &Job{ ID: jobID, - Type: JobTypeDelete, SnapshotID: snapshotID, - BlockNums: blockNums, StartTime: time.Now(), } - // Store job ID in snapshot descriptor if err := w.registry.SetRefCountJobID(ctx, snapshotID, jobID); err != nil { return 0, fmt.Errorf("failed to set job ID: %w", err) } select { case w.jobQueue <- job: - log.Printf("[snapshot/worker] Queued delete job %d for snapshot %d (%d blocks)", - jobID, snapshotID, len(blockNums)) + log.Printf("[snapshot/worker] Queued delete job %d for snapshot %d", jobID, snapshotID) return jobID, nil - default: - return 0, fmt.Errorf("job queue full") + case <-w.stopChan: + return 0, ErrWorkerStopped } } @@ -242,21 +176,10 @@ func (w *Worker) workerLoop(workerID int) { } } -// processJob executes a single refcount job. +// processJob executes a single job. func (w *Worker) processJob(job *Job) *JobResult { - ctx := context.Background() start := time.Now() - - var err error - switch job.Type { - case JobTypeCreate: - err = w.processCreateJob(ctx, job) - case JobTypeDelete: - err = w.processDeleteJob(ctx, job) - default: - err = fmt.Errorf("unknown job type: %d", job.Type) - } - + err := w.processDeleteJob(context.Background(), job) return &JobResult{ JobID: job.ID, SnapshotID: job.SnapshotID, @@ -266,128 +189,80 @@ func (w *Worker) processJob(job *Job) *JobResult { } } -// processCreateJob increments refcounts and transitions snapshot to Active. -func (w *Worker) processCreateJob(ctx context.Context, job *Job) error { - if len(job.BlockNums) == 0 { - // No blocks to process, just transition state - return w.registry.UpdateState(ctx, job.SnapshotID, StateActive) - } - - // Increment refcounts in batches for efficiency - const batchSize = 512 // Process 512 blocks per batch - for i := 0; i < len(job.BlockNums); i += batchSize { - end := i + batchSize - if end > len(job.BlockNums) { - end = len(job.BlockNums) - } - - batch := job.BlockNums[i:end] - if _, err := w.refRegion.IncrementBatch(ctx, batch); err != nil { - return fmt.Errorf("failed to increment refcounts (batch %d-%d): %w", i, end, err) - } - } - - // Transition snapshot to Active state - if err := w.registry.UpdateState(ctx, job.SnapshotID, StateActive); err != nil { - return fmt.Errorf("failed to update snapshot state: %w", err) - } - - log.Printf("[snapshot/worker] Snapshot %d now active (%d blocks processed)", - job.SnapshotID, len(job.BlockNums)) - return nil -} - -// processDeleteJob handles snapshot deletion with merge-up for middle snapshots. -// -// For leaf snapshots (no children): -// - Decrement all block refcounts -// - Free blocks where refcount reaches 0 -// -// For middle snapshots (has children): -// - Re-parent children to this snapshot's parent (merge-up) -// - Decrement refcounts (shared blocks won't reach 0) -// - Free blocks that reach refcount 0 +// processDeleteJob re-parents the snapshot's children to its parent, gives +// back the blocks it captured, and removes its descriptor. func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { - // First, get the snapshot to find its parent snap, err := w.registry.Get(ctx, job.SnapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + return nil + } if err != nil { return fmt.Errorf("failed to get snapshot %d: %w", job.SnapshotID, err) } + if snap.State != StateDeleting { + return fmt.Errorf("snapshot %d is %s, not deleting: %w", job.SnapshotID, snap.State, ErrSnapshotInvalidState) + } - // Check for children (determines if this is a leaf or middle snapshot) children, err := w.registry.GetChildren(ctx, job.SnapshotID) if err != nil { return fmt.Errorf("failed to get children for snapshot %d: %w", job.SnapshotID, err) } - - // Handle merge-up: re-parent children to grandparent - if len(children) > 0 { - log.Printf("[snapshot/worker] Snapshot %d has %d children, performing merge-up to parent %d", - job.SnapshotID, len(children), snap.ParentID) - - for _, child := range children { - if err := w.registry.UpdateParentID(ctx, child.SnapshotID, snap.ParentID); err != nil { - return fmt.Errorf("failed to re-parent snapshot %d: %w", child.SnapshotID, err) - } - log.Printf("[snapshot/worker] Re-parented snapshot %d from %d to %d", - child.SnapshotID, job.SnapshotID, snap.ParentID) + for _, child := range children { + if err := w.registry.UpdateParentID(ctx, child.SnapshotID, snap.ParentID); err != nil { + return fmt.Errorf("failed to re-parent snapshot %d: %w", child.SnapshotID, err) } + log.Printf("[snapshot/worker] Re-parented snapshot %d from %d to %d", + child.SnapshotID, job.SnapshotID, snap.ParentID) } - // Decrement refcounts and free blocks that reach 0 - var freedCount int - if len(job.BlockNums) > 0 { - freedCount, err = w.decrementAndFreeBlocks(ctx, job.BlockNums) - if err != nil { - return fmt.Errorf("failed to decrement/free blocks: %w", err) + if snap.HasData() { + if err := w.ReleaseData(ctx, snap.DataInode); err != nil { + return fmt.Errorf("failed to release data of snapshot %d: %w", job.SnapshotID, err) } } - // Remove the snapshot descriptor if err := w.registry.Remove(ctx, job.SnapshotID); err != nil { return fmt.Errorf("failed to remove snapshot: %w", err) } - log.Printf("[snapshot/worker] Snapshot %d deleted (%d blocks processed, %d freed)", - job.SnapshotID, len(job.BlockNums), freedCount) + // Freed only once no descriptor names it, so a reused slot is never released again. + if snap.HasData() { + if err := w.inodeTable.Free(ctx, snap.DataInode); err != nil { + log.Printf("[snapshot/worker] Failed to free data inode %d of snapshot %d: %v", + snap.DataInode, job.SnapshotID, err) + } + } + + log.Printf("[snapshot/worker] Snapshot %d deleted", job.SnapshotID) return nil } -// decrementAndFreeBlocks decrements refcounts for blocks and frees those that reach 0. -// Returns the count of blocks freed. -func (w *Worker) decrementAndFreeBlocks(ctx context.Context, blockNums []uint64) (int, error) { - const batchSize = 512 - freedCount := 0 - - for i := 0; i < len(blockNums); i += batchSize { - end := i + batchSize - if end > len(blockNums) { - end = len(blockNums) - } - - batch := blockNums[i:end] - newCounts, err := w.refRegion.DecrementBatch(ctx, batch) +// ReleaseData gives back the blocks held by a snapshot's data inode. The empty +// extent map is committed before any block is released, so a crash leaks +// blocks at worst and calling it again is safe. +func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { + const attempts = 5 + for attempt := 1; ; attempt++ { + ino, err := w.inodeTable.Read(num) if err != nil { - return freedCount, fmt.Errorf("failed to decrement refcounts (batch %d-%d): %w", i, end, err) + return err } - - // Free blocks that reached refcount 0 - if w.bitmapMgr != nil { - for j, newCount := range newCounts { - if newCount == 0 { - blockNum := batch[j] - if err := w.bitmapMgr.Free(ctx, blockNum); err != nil { - // Log but don't fail - block leak is better than corruption - log.Printf("[snapshot/worker] Warning: failed to free block %d: %v", blockNum, err) - } else { - freedCount++ - } - } - } + if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { + return nil + } + if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil { + w.extentMgr.DiscardPending(ctx, ino) + return err + } + err = w.inodeTable.Write(ctx, ino) + if err == nil { + return nil + } + w.extentMgr.DiscardPending(ctx, ino) + if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts { + return err } } - - return freedCount, nil } // resultHandler processes completed job results. @@ -416,83 +291,44 @@ func (w *Worker) resultHandler() { } } -// RecoverIncompleteJobs scans for snapshots in StateCreating or StateDeleting -// and re-queues their background jobs. Call this on mount/startup. +// RecoverIncompleteJobs deletes snapshots that a crash left creating or +// deleting. A snapshot still creating was never reported as created, so it is +// rolled back, unless another node activates it first. Call this on mount, +// after Start. func (w *Worker) RecoverIncompleteJobs(ctx context.Context) error { w.mu.Lock() defer w.mu.Unlock() - // Scan registry for incomplete snapshots - stats, err := w.registry.Scan(ctx) - if err != nil { - return fmt.Errorf("failed to scan registry: %w", err) - } - - if stats.CreatingCount == 0 && stats.DeletingCount == 0 { - log.Printf("[snapshot/worker] No incomplete jobs to recover") - return nil - } - - log.Printf("[snapshot/worker] Recovering %d creating + %d deleting snapshots", - stats.CreatingCount, stats.DeletingCount) - - // Get all snapshots and process incomplete ones snapshots, err := w.registry.List(ctx) if err != nil { return fmt.Errorf("failed to list snapshots: %w", err) } + var recovered int for _, snap := range snapshots { switch snap.State { case StateCreating: - if err := w.recoverCreateJob(ctx, snap); err != nil { - log.Printf("[snapshot/worker] Failed to recover create job for snapshot %d: %v", - snap.SnapshotID, err) + if err := w.registry.Abandon(ctx, snap.SnapshotID); err != nil { + log.Printf("[snapshot/worker] Not rolling back snapshot %d: %v", snap.SnapshotID, err) + continue } - case StateDeleting: - if err := w.recoverDeleteJob(ctx, snap); err != nil { - log.Printf("[snapshot/worker] Failed to recover delete job for snapshot %d: %v", - snap.SnapshotID, err) - } + default: + continue } + if _, err := w.QueueDeleteJob(ctx, snap.SnapshotID); err != nil { + log.Printf("[snapshot/worker] Failed to queue cleanup of snapshot %d: %v", snap.SnapshotID, err) + continue + } + recovered++ } - return nil -} - -// recoverCreateJob recovers a snapshot stuck in StateCreating. -func (w *Worker) recoverCreateJob(ctx context.Context, snap *Descriptor) error { - // Read the inode to get current block list - ino, err := w.inodeTable.Read(snap.TargetInode) - if err != nil { - return fmt.Errorf("failed to read inode %d: %w", snap.TargetInode, err) - } - - // Collect all block numbers from the inode - blockNums, err := w.extentMgr.DataBlocks(ino) - if err != nil { - return fmt.Errorf("failed to list blocks of inode %d: %w", snap.TargetInode, err) + if recovered == 0 { + log.Printf("[snapshot/worker] No incomplete jobs to recover") + } else { + log.Printf("[snapshot/worker] Cleaning up %d incomplete snapshot(s)", recovered) } - - // Re-queue the create job - // Note: This may result in some blocks being incremented twice if crash - // happened mid-way. This is safe - it means slightly higher refcounts - // that will be cleaned up when the snapshot is deleted. - _, err = w.QueueCreateJob(ctx, snap.SnapshotID, snap.TargetInode, blockNums) - return err -} - -// recoverDeleteJob recovers a snapshot stuck in StateDeleting. -func (w *Worker) recoverDeleteJob(ctx context.Context, snap *Descriptor) error { - // For deletion, we need to read the snapshot's extent tree (not the current inode) - // Since we stored ExtentTreeRoot at snapshot time, we use that. - // However, the blocks may have already been decremented, so we need to be careful. - - // For now, just remove the descriptor - the refcounts were either - // already decremented or will be off by 1 (leak) which is acceptable - // for crash recovery. - return w.registry.Remove(ctx, snap.SnapshotID) + return nil } // PendingJobs returns the number of jobs waiting in the queue. diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index 0426afb1..fbda8138 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -2,12 +2,14 @@ package snapshot import ( "context" + "errors" "fmt" "os" "path/filepath" "testing" "time" + "github.com/supporttools/GoClusterFS/internal/bitmap" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" "github.com/supporttools/GoClusterFS/internal/refcount" @@ -18,11 +20,12 @@ const ( testBlockSize = 1024 * 1024 // 1MB testDeviceSize = 100 * testBlockSize testDataBlocks = 50 + testBitmapStart = uint64(60 * testBlockSize) + testInodeStart = uint64(70 * testBlockSize) + testInodeSize = uint64(10 * testBlockSize) testRefcountStart = uint64(80 * testBlockSize) testSnapshotStart = uint64(90 * testBlockSize) testSnapshotSize = uint64(5 * 4096) // 5 sectors - testInodeStart = uint64(70 * testBlockSize) - testInodeSize = uint64(10 * testBlockSize) ) // setupTestDevice creates a temporary device for testing. @@ -61,73 +64,169 @@ func setupTestDevice(t *testing.T) (*storage.Device, string, func()) { return dev, devPath, cleanup } -// setupTestComponents creates all components needed for worker testing. -const jobTimeout = 10 * time.Second +// workerEnv holds a small filesystem: data blocks start at device offset 0. +type workerEnv struct { + t *testing.T + ctx context.Context + reg *Registry + refs *refcount.Region + bm *bitmap.Region + tbl *inode.Table + ext *extent.Manager + w *Worker + done chan *JobResult +} -// waitForJob polls done until the worker's background job has finished. -func waitForJob(t *testing.T, done func() bool) { +func newWorkerEnv(t *testing.T) *workerEnv { t.Helper() - deadline := time.Now().Add(jobTimeout) - for !done() { - if time.Now().After(deadline) { - t.Fatalf("Background job did not finish within %v", jobTimeout) - } - time.Sleep(10 * time.Millisecond) - } + dev, _, cleanup := setupTestDevice(t) + t.Cleanup(cleanup) + + var nodeUUID [16]byte + env := &workerEnv{ + t: t, + ctx: context.Background(), + reg: NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil), + refs: refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil), + bm: bitmap.NewRegion(dev, testBitmapStart, testDataBlocks, nil, nodeUUID), + tbl: inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID), + done: make(chan *JobResult, 64), + } + if err := env.reg.Initialize(env.ctx); err != nil { + t.Fatalf("initialize registry: %v", err) + } + env.ext = extent.NewManager(dev, env.bm, nil, 0) + env.ext.SetRefcounts(env.refs) + env.tbl.SetTreeFlusher(env.ext) + + cfg := DefaultWorkerConfig() + cfg.OnJobComplete = func(r *JobResult) { env.done <- r } + env.w = NewWorker(env.reg, env.ext, env.tbl, cfg) + return env } -func snapshotActive(reg *Registry, id uint64) func() bool { - return func() bool { - desc, err := reg.Get(context.Background(), id) - return err == nil && desc.State == StateActive - } +func (env *workerEnv) start() { + env.w.Start() + env.t.Cleanup(env.w.Stop) } -func snapshotRemoved(reg *Registry, id uint64) func() bool { - return func() bool { - _, err := reg.Get(context.Background(), id) - return err == ErrSnapshotNotFound +func (env *workerEnv) claim() *inode.Inode { + env.t.Helper() + num, err := env.tbl.FindFreeSlot() + if err != nil { + env.t.Fatal(err) + } + ino, err := env.tbl.ClaimSlot(env.ctx, num) + if err != nil { + env.t.Fatal(err) } + ino.Mode = inode.S_IFREG | 0o644 + return ino } -func setupTestComponents(t *testing.T, dev *storage.Device) (*Registry, *refcount.Region, *extent.Manager, *inode.Table) { - t.Helper() - - // Create refcount region - refReg := refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil) +func (env *workerEnv) write(ino *inode.Inode) { + env.t.Helper() + if err := env.tbl.Write(env.ctx, ino); err != nil { + env.t.Fatalf("write inode %d: %v", ino.InodeNum, err) + } +} - // Initialize refcount region - ctx := context.Background() - if err := refReg.Initialize(ctx); err != nil { - t.Fatalf("Failed to initialize refcount region: %v", err) +func (env *workerEnv) file(blocks uint64) (*inode.Inode, []uint64) { + env.t.Helper() + ino := env.claim() + if _, err := env.ext.AllocateBlocks(env.ctx, ino, 0, blocks); err != nil { + env.t.Fatal(err) } + env.write(ino) + data, err := env.ext.DataBlocks(ino) + if err != nil { + env.t.Fatal(err) + } + return ino, data +} - // Create snapshot registry - snapReg := NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil) +// snapshot captures file the way createSnapshot does and leaves it in state. +func (env *workerEnv) snapshot(file *inode.Inode, name string, state State) *Descriptor { + env.t.Helper() + data := env.claim() + data.Flags |= inode.FlagSnapshotData + env.write(data) + desc, err := env.reg.Create(env.ctx, name, file.InodeNum, 0, data.InodeNum, file.AllocatedBlocks, file.Size, FlagReadOnly) + if err != nil { + env.t.Fatal(err) + } + blocks, err := env.ext.DataBlocks(file) + if err != nil { + env.t.Fatal(err) + } + if err := env.refs.Share(env.ctx, blocks); err != nil { + env.t.Fatal(err) + } + if err := env.ext.CopyExtentsForClone(env.ctx, file, data); err != nil { + env.t.Fatal(err) + } + env.write(data) + if err := env.ext.MarkAllShared(file); err != nil { + env.t.Fatal(err) + } + env.write(file) + if state != StateCreating { + if err := env.reg.UpdateState(env.ctx, desc.SnapshotID, state); err != nil { + env.t.Fatal(err) + } + } + return desc +} - // Initialize snapshot registry - if err := snapReg.Initialize(ctx); err != nil { - t.Fatalf("Failed to initialize snapshot registry: %v", err) +// remove marks a snapshot deleting, runs its delete job, and waits for it. +func (env *workerEnv) remove(id uint64) { + env.t.Helper() + if err := env.reg.Delete(env.ctx, id); err != nil { + env.t.Fatal(err) } + if _, err := env.w.QueueDeleteJob(env.ctx, id); err != nil { + env.t.Fatal(err) + } + env.wait(1) +} - // Create extent manager (bitmap can be nil for tests not needing allocation) - extMgr := extent.NewManager(dev, nil, nil, 0) +func (env *workerEnv) wait(jobs int) { + env.t.Helper() + timeout := time.After(10 * time.Second) + for range jobs { + select { + case r := <-env.done: + if !r.Success { + env.t.Fatalf("job %d for snapshot %d failed: %v", r.JobID, r.SnapshotID, r.Error) + } + case <-timeout: + env.t.Fatalf("timed out waiting for %d snapshot job(s)", jobs) + } + } +} - // Create inode table - var nodeUUID [16]byte - inodeTbl := inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID) +func (env *workerEnv) allocated(block uint64) bool { + env.t.Helper() + ok, err := env.bm.IsAllocated(block) + if err != nil { + env.t.Fatal(err) + } + return ok +} - return snapReg, refReg, extMgr, inodeTbl +func (env *workerEnv) checkGone(desc *Descriptor) { + env.t.Helper() + if _, err := env.reg.Get(env.ctx, desc.SnapshotID); !errors.Is(err, ErrSnapshotNotFound) { + env.t.Errorf("snapshot %q still listed: %v", desc.GetName(), err) + } + if ino, err := env.tbl.Read(desc.DataInode); err != nil || ino.Magic != 0 { + env.t.Errorf("data inode %d of snapshot %q not freed (err=%v)", desc.DataInode, desc.GetName(), err) + } } func TestWorkerStartStop(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) + env := newWorkerEnv(t) + worker := env.w if worker.IsRunning() { t.Error("Worker should not be running before Start()") @@ -156,242 +255,75 @@ func TestWorkerStartStop(t *testing.T) { } } -func TestWorkerQueueCreateJob(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create a snapshot first - desc, err := snapReg.Create(ctx, "test-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - cfg.OnJobComplete = func(result *JobResult) { - t.Logf("Job %d completed: success=%v, duration=%v", result.JobID, result.Success, result.Duration) - } - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue a create job - blockNums := []uint64{10, 11, 12} - jobID, err := worker.QueueCreateJob(ctx, desc.SnapshotID, 100, blockNums) - if err != nil { - t.Fatalf("Failed to queue create job: %v", err) - } - - if jobID == 0 { - t.Error("Job ID should not be 0") - } - - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) - - // Verify refcounts were incremented - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 1 { - t.Errorf("Block %d refcount = %d, want 1", blockNum, entry.RefCount) - } - } - - // Verify snapshot state changed to Active - updatedDesc, err := snapReg.Get(ctx, desc.SnapshotID) - if err != nil { - t.Fatalf("Failed to get snapshot: %v", err) - } - if updatedDesc.State != StateActive { - t.Errorf("Snapshot state = %v, want StateActive", updatedDesc.State) - } -} - func TestWorkerQueueDeleteJob(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create and complete a snapshot first - desc, err := snapReg.Create(ctx, "test-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } + env := newWorkerEnv(t) + env.start() + initial := env.bm.FreeCount() - // Set up initial refcounts - blockNums := []uint64{20, 21, 22} - for _, blockNum := range blockNums { - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount for block %d: %v", blockNum, err) - } - } + file, blocks := env.file(3) + snap := env.snapshot(file, "test-snap", StateActive) - // Mark snapshot as active, then delete - if err := snapReg.UpdateState(ctx, desc.SnapshotID, StateActive); err != nil { - t.Fatalf("Failed to update state: %v", err) + // The file lets go of its last two blocks; only the snapshot holds them now. + if err := env.ext.Truncate(env.ctx, file, extent.BlockSize); err != nil { + t.Fatal(err) } - if err := snapReg.Delete(ctx, desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot: %v", err) + env.write(file) + if !env.allocated(blocks[1]) || !env.allocated(blocks[2]) { + t.Fatal("truncating the file freed blocks the snapshot holds") } - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job - jobID, err := worker.QueueDeleteJob(ctx, desc.SnapshotID, blockNums) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - if jobID == 0 { - t.Error("Job ID should not be 0") + env.remove(snap.SnapshotID) + env.checkGone(snap) + if env.allocated(blocks[1]) || env.allocated(blocks[2]) { + t.Error("blocks only the snapshot held stay allocated") } - - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) - - // Verify refcounts were decremented - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 0 { - t.Errorf("Block %d refcount = %d, want 0", blockNum, entry.RefCount) - } + if !env.allocated(blocks[0]) { + t.Error("deleting the snapshot freed a block the file still maps") } - - // Verify snapshot was removed - _, err = snapReg.Get(ctx, desc.SnapshotID) - if err != ErrSnapshotNotFound { - t.Errorf("Snapshot should be removed, got err = %v", err) - } -} - -func TestWorkerNotRunning(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - // Don't start the worker - - _, err := worker.QueueCreateJob(ctx, 1, 100, []uint64{1, 2, 3}) - if err == nil { - t.Error("QueueCreateJob should fail when worker not running") - } - - _, err = worker.QueueDeleteJob(ctx, 1, []uint64{1, 2, 3}) - if err == nil { - t.Error("QueueDeleteJob should fail when worker not running") - } -} - -func TestWorkerEmptyBlockList(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create snapshot - desc, err := snapReg.Create(ctx, "empty-snap", 100, 0, 0, 0, 0, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue create job with empty block list - _, err = worker.QueueCreateJob(ctx, desc.SnapshotID, 100, []uint64{}) - if err != nil { - t.Fatalf("Failed to queue empty create job: %v", err) - } - - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) - - // Verify snapshot state changed to Active - updatedDesc, err := snapReg.Get(ctx, desc.SnapshotID) - if err != nil { - t.Fatalf("Failed to get snapshot: %v", err) - } - if updatedDesc.State != StateActive { - t.Errorf("Snapshot state = %v, want StateActive", updatedDesc.State) + if got := env.bm.FreeCount(); got != initial-1 { + t.Errorf("free blocks = %d, want %d", got, initial-1) } } // TestWorkerQueuesJobsWithCanceledContext checks that a canceled FUSE request // context cannot drop a job for a snapshot that already exists. func TestWorkerQueuesJobsWithCanceledContext(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, DefaultWorkerConfig()) - worker.Start() - defer worker.Stop() + env := newWorkerEnv(t) + file, _ := env.file(1) + var snaps []*Descriptor + for i := range 20 { + snaps = append(snaps, env.snapshot(file, fmt.Sprintf("snap%d", i), StateDeleting)) + } + env.start() - canceled, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(env.ctx) cancel() - - for i := 0; i < 20; i++ { - desc, err := snapReg.Create(context.Background(), fmt.Sprintf("snap-%d", i), 100, 0, 0, 0, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot %d: %v", i, err) - } - - if _, err := worker.QueueCreateJob(canceled, desc.SnapshotID, 100, []uint64{}); err != nil { - t.Fatalf("QueueCreateJob %d with canceled context: %v", i, err) + for _, snap := range snaps { + if _, err := env.w.QueueDeleteJob(ctx, snap.SnapshotID); err != nil { + t.Fatalf("queue delete of %q: %v", snap.GetName(), err) } - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) + } + env.wait(len(snaps)) + for _, snap := range snaps { + env.checkGone(snap) + } +} - if err := snapReg.Delete(context.Background(), desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot %d: %v", i, err) - } - if _, err := worker.QueueDeleteJob(canceled, desc.SnapshotID, []uint64{}); err != nil { - t.Fatalf("QueueDeleteJob %d with canceled context: %v", i, err) - } - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) +func TestWorkerNotRunning(t *testing.T) { + env := newWorkerEnv(t) + if _, err := env.w.QueueDeleteJob(env.ctx, 1); !errors.Is(err, ErrWorkerStopped) { + t.Errorf("QueueDeleteJob on a stopped worker: got %v, want ErrWorkerStopped", err) } } func TestWorkerPendingJobs(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - cfg := DefaultWorkerConfig() - cfg.QueueSize = 10 - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - - // Before starting, pending should be 0 - if worker.PendingJobs() != 0 { - t.Errorf("PendingJobs = %d before start, want 0", worker.PendingJobs()) + env := newWorkerEnv(t) + if env.w.PendingJobs() != 0 { + t.Errorf("PendingJobs = %d before start, want 0", env.w.PendingJobs()) } - - worker.Start() - defer worker.Stop() - - // After starting with no jobs, still 0 - if worker.PendingJobs() != 0 { - t.Errorf("PendingJobs = %d after start, want 0", worker.PendingJobs()) + env.start() + if env.w.PendingJobs() != 0 { + t.Errorf("PendingJobs = %d after start, want 0", env.w.PendingJobs()) } } @@ -406,83 +338,70 @@ func TestDefaultWorkerConfig(t *testing.T) { } } -// TestLeafSnapshotDeletion tests deleting a snapshot with no children. -// Expected behavior: -// - Refcounts decremented -// - Blocks with refcount=0 are freed via bitmap -// - Snapshot descriptor removed +// TestLeafSnapshotDeletion deletes the only owner left after the file is gone. func TestLeafSnapshotDeletion(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + env.start() + initial := env.bm.FreeCount() - // Create a snapshot (will be a leaf since it has no children) - desc, err := snapReg.Create(ctx, "leaf-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) + file, blocks := env.file(2) + snap := env.snapshot(file, "leaf-snap", StateActive) + if err := env.ext.Delete(env.ctx, file); err != nil { + t.Fatal(err) } - - // Set up initial refcounts (simulating blocks used by snapshot) - blockNums := []uint64{30, 31, 32} - for _, blockNum := range blockNums { - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount for block %d: %v", blockNum, err) + if err := env.tbl.Free(env.ctx, file.InodeNum); err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if !env.allocated(b) { + t.Fatalf("deleting the file freed block %d that the snapshot holds", b) } } - // Mark as active then delete - if err := snapReg.UpdateState(ctx, desc.SnapshotID, StateActive); err != nil { - t.Fatalf("Failed to update state: %v", err) - } - if err := snapReg.Delete(ctx, desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot: %v", err) + env.remove(snap.SnapshotID) + env.checkGone(snap) + if got := env.bm.FreeCount(); got != initial { + t.Errorf("free blocks = %d, want %d", got, initial) } +} - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() +func TestRecoverIncompleteSnapshots(t *testing.T) { + env := newWorkerEnv(t) + file, blocks := env.file(2) + active := env.snapshot(file, "active", StateActive) + creating := env.snapshot(file, "creating", StateCreating) + deleting := env.snapshot(file, "deleting", StateDeleting) + env.start() - // Queue delete job - _, err = worker.QueueDeleteJob(ctx, desc.SnapshotID, blockNums) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) + if err := env.w.RecoverIncompleteJobs(env.ctx); err != nil { + t.Fatal(err) } + env.wait(2) - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) - - // Verify refcounts are 0 (all decremented) - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) + env.checkGone(creating) + env.checkGone(deleting) + if got, err := env.reg.Get(env.ctx, active.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("active snapshot after recovery: %+v, %v", got, err) + } + for _, b := range blocks { + e, err := env.refs.Get(env.ctx, b) if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue + t.Fatal(err) } - if entry.RefCount != 0 { - t.Errorf("Block %d refcount = %d, want 0", blockNum, entry.RefCount) + if !env.allocated(b) || e.RefCount != 2 { + t.Errorf("block %d: allocated=%v refcount=%d, want held by the file and one snapshot", b, env.allocated(b), e.RefCount) } } - - // Verify snapshot is removed - _, err = snapReg.Get(ctx, desc.SnapshotID) - if err != ErrSnapshotNotFound { - t.Errorf("Expected ErrSnapshotNotFound, got %v", err) - } } // TestMiddleSnapshotDeletionMergeUp tests deleting a snapshot that has children. // Expected behavior: // - Children are re-parented to grandparent (merge-up) -// - Refcounts decremented // - Snapshot descriptor removed func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx + env.start() // Create parent snapshot (root) rootSnap, err := snapReg.Create(ctx, "root-snap", 100, 0, 0, 2, 1024, FlagReadOnly) @@ -511,35 +430,7 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { t.Fatalf("Failed to activate leaf: %v", err) } - // Set up refcounts for middle snapshot's blocks - middleBlocks := []uint64{40, 41, 42} - for _, blockNum := range middleBlocks { - // Increment twice: once for middle, once simulating shared with leaf - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount: %v", err) - } - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount: %v", err) - } - } - - // Delete middle snapshot - if err := snapReg.Delete(ctx, middleSnap.SnapshotID); err != nil { - t.Fatalf("Failed to delete middle snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job for middle snapshot - _, err = worker.QueueDeleteJob(ctx, middleSnap.SnapshotID, middleBlocks) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - waitForJob(t, snapshotRemoved(snapReg, middleSnap.SnapshotID)) + env.remove(middleSnap.SnapshotID) // Verify leaf snapshot was re-parented to root updatedLeaf, err := snapReg.Get(ctx, leafSnap.SnapshotID) @@ -551,18 +442,6 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { updatedLeaf.ParentID, rootSnap.SnapshotID) } - // Verify refcounts decremented (should be 1 now, not 0, due to leaf sharing) - for _, blockNum := range middleBlocks { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 1 { - t.Errorf("Block %d refcount = %d, want 1 (shared with leaf)", blockNum, entry.RefCount) - } - } - // Verify middle snapshot is removed _, err = snapReg.Get(ctx, middleSnap.SnapshotID) if err != ErrSnapshotNotFound { @@ -582,11 +461,8 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { // TestGetChildren verifies the GetChildren registry method. func TestGetChildren(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, _, _, _ := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx // Create parent snapshot parent, err := snapReg.Create(ctx, "parent", 100, 0, 0, 1, 100, FlagReadOnly) @@ -646,11 +522,8 @@ func TestGetChildren(t *testing.T) { // TestUpdateParentID verifies the UpdateParentID registry method. func TestUpdateParentID(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, _, _, _ := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx // Create a chain: grandparent -> parent -> child grandparent, err := snapReg.Create(ctx, "grandparent", 100, 0, 0, 1, 100, FlagReadOnly) @@ -697,11 +570,9 @@ func TestUpdateParentID(t *testing.T) { // TestMultipleChildrenMergeUp tests merge-up when a snapshot has multiple children. func TestMultipleChildrenMergeUp(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx + env.start() // Create: root -> middle -> {child1, child2, child3} root, err := snapReg.Create(ctx, "root", 100, 0, 0, 1, 100, FlagReadOnly) @@ -732,23 +603,7 @@ func TestMultipleChildrenMergeUp(t *testing.T) { children = append(children, child) } - // Delete middle - if err := snapReg.Delete(ctx, middle.SnapshotID); err != nil { - t.Fatalf("Failed to delete middle: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job - _, err = worker.QueueDeleteJob(ctx, middle.SnapshotID, []uint64{}) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - waitForJob(t, snapshotRemoved(snapReg, middle.SnapshotID)) + env.remove(middle.SnapshotID) // Verify all children now point to root for i, child := range children { @@ -769,3 +624,33 @@ func TestMultipleChildrenMergeUp(t *testing.T) { t.Errorf("Middle snapshot should be removed, got: %v", err) } } + +// A delete job for a snapshot that is not marked deleting, such as one another +// node activated after a mount listed it as creating, must leave it alone. +func TestDeleteJobRequiresDeletingState(t *testing.T) { + env := newWorkerEnv(t) + env.start() + file, blocks := env.file(1) + snap := env.snapshot(file, "active", StateActive) + + if _, err := env.w.QueueDeleteJob(env.ctx, snap.SnapshotID); err != nil { + t.Fatal(err) + } + select { + case r := <-env.done: + if r.Success { + t.Fatal("delete job succeeded on an active snapshot") + } + case <-time.After(10 * time.Second): + t.Fatal("delete job did not finish") + } + if got, err := env.reg.Get(env.ctx, snap.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("snapshot after refused delete: %+v, %v; want active", got, err) + } + if ino, err := env.tbl.Read(snap.DataInode); err != nil || ino.Magic == 0 { + t.Fatalf("data inode %d freed by a refused delete (err=%v)", snap.DataInode, err) + } + if e, err := env.refs.Get(env.ctx, blocks[0]); err != nil || e.RefCount != 2 { + t.Fatalf("block refcount %+v, %v; want 2", e, err) + } +} diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index e75321e2..6c71e0f1 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -24,11 +24,13 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o goclusterfs-mount ./cmd/mount FROM debian:bookworm-slim AS runtime # Install runtime dependencies: +# - attr: setfattr/getfattr for snapshot and clone tests # - fuse3: FUSE filesystem support # - iproute2: tc (traffic control) for network simulation # - iptables: Firewall rules for network partition simulation # - procps: ps, top for debugging RUN apt-get update && apt-get install -y --no-install-recommends \ + attr \ fuse3 \ iproute2 \ iptables \ diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index adca04ae..ba4fda7b 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2796,6 +2796,199 @@ test_extent_tree_interleaved_appends() { return 0 } +md5_on_node_matches() { + local node=$1 path=$2 want=$3 got + got=$(exec_on_node "$node" sh -c "md5sum < $path | cut -d' ' -f1") + if [ "$got" != "$want" ]; then + log "ERROR: $node $path md5 $got, want $want" + return 1 + fi +} + +# Snapshots a file, then overwrites it in place, appends to it and truncates +# it. A clone of the snapshot must hold the original data on both nodes. The +# expected data is built with the same edits on a local copy. +test_snapshot_preserves_data() { + log "Testing that a snapshot keeps its data..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]:-${NODE_ARRAY[0]}}" + local file=/mnt/goclusterfs/snap_data.bin + local clone=/mnt/goclusterfs/snap_data_clone.bin + + exec_on_node "$writer" rm -f "$file" "$clone" + exec_on_node "$writer" sh -c "head -c 3145728 /dev/urandom > /tmp/snap_orig && cp /tmp/snap_orig /tmp/snap_live && cp /tmp/snap_orig $file" || return 1 + exec_on_node "$writer" setfattr -n user.snapshot -v snap_data_before "$file" || { + log "ERROR: snapshot failed" + return 1 + } + exec_on_node "$writer" sh -c "head -c 4096 /dev/urandom > /tmp/snap_patch && head -c 2097152 /dev/urandom > /tmp/snap_tail" + for target in "$file" /tmp/snap_live; do + exec_on_node "$writer" sh -c "dd if=/tmp/snap_patch of=$target bs=4096 seek=257 conv=notrunc 2>/dev/null && cat /tmp/snap_tail >> $target && truncate -s 1572864 $target" || { + log "ERROR: changing $target failed" + return 1 + } + done + exec_on_node "$writer" setfattr -n user.snapshot.clone -v "snap_data_before:/snap_data_clone.bin" "$file" || { + log "ERROR: clone from snapshot failed" + return 1 + } + + local want_clone want_live + want_clone=$(exec_on_node "$writer" sh -c "md5sum < /tmp/snap_orig | cut -d' ' -f1") + want_live=$(exec_on_node "$writer" sh -c "md5sum < /tmp/snap_live | cut -d' ' -f1") + for node in $(printf '%s\n' "$reader" "$writer" | uniq); do + restart_node "$node" + wait_node_healthy "$node" 30 || return 1 + md5_on_node_matches "$node" "$clone" "$want_clone" || return 1 + md5_on_node_matches "$node" "$file" "$want_live" || return 1 + done + + exec_on_node "$writer" setfattr -x user.snapshot.snap_data_before "$file" + log "Snapshot keeps its data test passed" + return 0 +} + +# Writes into a clone and into its source with dd conv=notrunc. Each write must +# change only the file written, on both nodes. +test_clone_in_place_write() { + log "Testing in-place writes to a clone and its source..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]:-${NODE_ARRAY[0]}}" + local src=/mnt/goclusterfs/clone_src.bin + local clone=/mnt/goclusterfs/clone_inplace.bin + + exec_on_node "$writer" rm -f "$src" "$clone" + exec_on_node "$writer" sh -c "head -c 2097152 /dev/urandom > /tmp/clone_src && cp /tmp/clone_src /tmp/clone_copy && cp /tmp/clone_src $src" || return 1 + exec_on_node "$writer" setfattr -n user.clone -v /clone_inplace.bin "$src" || { + log "ERROR: clone failed" + return 1 + } + exec_on_node "$writer" sh -c "head -c 4096 /dev/urandom > /tmp/clone_patch_a && head -c 4096 /dev/urandom > /tmp/clone_patch_b" + exec_on_node "$writer" sh -c "dd if=/tmp/clone_patch_a of=$clone bs=4096 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_a of=/tmp/clone_copy bs=4096 conv=notrunc 2>/dev/null" || return 1 + exec_on_node "$writer" sh -c "dd if=/tmp/clone_patch_b of=$src bs=4096 seek=256 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_b of=/tmp/clone_src bs=4096 seek=256 conv=notrunc 2>/dev/null" || return 1 + + local want_src want_clone + want_src=$(exec_on_node "$writer" sh -c "md5sum < /tmp/clone_src | cut -d' ' -f1") + want_clone=$(exec_on_node "$writer" sh -c "md5sum < /tmp/clone_copy | cut -d' ' -f1") + for node in $(printf '%s\n' "$reader" "$writer" | uniq); do + restart_node "$node" + wait_node_healthy "$node" 30 || return 1 + md5_on_node_matches "$node" "$src" "$want_src" || return 1 + md5_on_node_matches "$node" "$clone" "$want_clone" || return 1 + done + + log "In-place clone write test passed" + return 0 +} + +# Snapshots a file, overwrites both of its blocks, then deletes the snapshot. +# The delete must finish and give back exactly the two blocks only the +# snapshot held. +test_snapshot_delete_after_overwrite() { + log "Testing snapshot delete after overwrite..." + + local node="${NODE_ARRAY[0]}" + local file=/mnt/goclusterfs/snap_delete.bin + local free_before free_now + + exec_on_node "$node" rm -f "$file" + exec_on_node "$node" sh -c "head -c 2097152 /dev/urandom > $file" || return 1 + free_before=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + exec_on_node "$node" setfattr -n user.snapshot -v snap_delete_temp "$file" || { + log "ERROR: snapshot failed" + return 1 + } + exec_on_node "$node" sh -c "head -c 2097152 /dev/urandom > /tmp/snap_new && dd if=/tmp/snap_new of=$file bs=1048576 conv=notrunc 2>/dev/null" || return 1 + free_now=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + if [ "$free_now" -ne $((free_before - 2)) ]; then + log "ERROR: $free_now free blocks after overwriting 2 snapshotted blocks, want $((free_before - 2))" + return 1 + fi + + exec_on_node "$node" setfattr -x user.snapshot.snap_delete_temp "$file" || { + log "ERROR: snapshot delete failed" + return 1 + } + local tries=0 + while exec_on_node "$node" sh -c "getfattr --only-values -n user.snapshots $file 2>/dev/null | grep -q snap_delete_temp"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log "ERROR: snapshot still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + free_now=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + if [ "$free_now" -ne "$free_before" ]; then + log "ERROR: $free_now free blocks after deleting the snapshot, want $free_before" + return 1 + fi + md5_on_node_matches "$node" "$file" "$(exec_on_node "$node" sh -c "md5sum < /tmp/snap_new | cut -d' ' -f1")" || return 1 + + log "Snapshot delete after overwrite test passed" + return 0 +} + +# Two nodes each take 10 snapshots of their own file at the same time. Each +# file must list all of its snapshots, and deleting one node's snapshots must +# leave the other node's alone. +test_snapshot_concurrent_creates() { + log "Testing concurrent snapshot creation on two nodes..." + + local node_a="${NODE_ARRAY[0]}" + local node_b="${NODE_ARRAY[1]:-}" + if [ -z "$node_b" ]; then + log "SKIP: needs two nodes" + return 0 + fi + local run=$$ + local file_a=/mnt/goclusterfs/conc_a.bin + local file_b=/mnt/goclusterfs/conc_b.bin + + exec_on_node "$node_a" sh -c "rm -f $file_a && head -c 1048576 /dev/urandom > $file_a" || return 1 + exec_on_node "$node_b" sh -c "rm -f $file_b && head -c 1048576 /dev/urandom > $file_b" || return 1 + + exec_on_node "$node_a" sh -c "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_a_${run}_\$i $file_a || exit 1; done" & + local pid_a=$! + exec_on_node "$node_b" sh -c "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_b_${run}_\$i $file_b || exit 1; done" & + local pid_b=$! + wait "$pid_a" || { log "ERROR: snapshots on $node_a failed"; return 1; } + wait "$pid_b" || { log "ERROR: snapshots on $node_b failed"; return 1; } + + local count_a count_b + count_a=$(exec_on_node "$node_a" sh -c "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -c ':active'") + count_b=$(exec_on_node "$node_b" sh -c "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_a" -ne 10 ] || [ "$count_b" -ne 10 ]; then + log "ERROR: $count_a and $count_b active snapshots, want 10 each" + return 1 + fi + + exec_on_node "$node_a" sh -c "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_a_${run}_\$i $file_a || exit 1; done" || { + log "ERROR: deleting snapshots on $node_a failed" + return 1 + } + local tries=0 + while exec_on_node "$node_a" sh -c "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -q conc_a_"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log "ERROR: $node_a snapshots still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + count_b=$(exec_on_node "$node_b" sh -c "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_b" -ne 10 ]; then + log "ERROR: $node_b lists $count_b snapshots after $node_a deleted its own, want 10" + return 1 + fi + + exec_on_node "$node_b" sh -c "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_b_${run}_\$i $file_b; done" + log "Concurrent snapshot creation test passed" + return 0 +} + run_test "Create and Read File" test_create_read_file run_test "Create Directory" test_create_directory run_test "File in Directory" test_file_in_directory @@ -2813,6 +3006,10 @@ run_test "Overwrite File" test_overwrite_file run_test "Sequential Access" test_sequential_access run_test "Extent Tree Sparse Writes" test_extent_tree_sparse_writes run_test "Extent Tree Interleaved Appends" test_extent_tree_interleaved_appends +run_test "Snapshot Preserves Data" test_snapshot_preserves_data +run_test "Clone In-Place Write" test_clone_in_place_write +run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite +run_test "Snapshot Concurrent Creates" test_snapshot_concurrent_creates if [ "$SKIP_CONCURRENT_TESTS" = "true" ]; then log "SKIPPING: Concurrent Writes (known race conditions - set SKIP_CONCURRENT_TESTS=false to enable)" diff --git a/tests/integration/extent_tree_test.go b/tests/integration/extent_tree_test.go index ede43b08..83fec21a 100644 --- a/tests/integration/extent_tree_test.go +++ b/tests/integration/extent_tree_test.go @@ -5,7 +5,6 @@ import ( "os" "syscall" "testing" - "time" "golang.org/x/sys/unix" ) @@ -220,12 +219,10 @@ func TestExtentTreeSnapshot(t *testing.T) { } t.Fatalf("Failed to create snapshot: %v", err) } - time.Sleep(500 * time.Millisecond) // background refcount job if err := unix.Setxattr(path, "user.snapshot.clone", []byte("tree-snap:/clone.bin"), 0); err != nil { t.Fatalf("clone from snapshot failed: %v", err) } - time.Sleep(500 * time.Millisecond) cloneWant := append([]byte(nil), want...) checkFile(t, mfs, "clone.bin", cloneWant) @@ -248,7 +245,7 @@ func TestExtentTreeSnapshot(t *testing.T) { f.Close() want = append(want, tail...) - // Blocks 4 and 5 are shared with the clone; only the new block 6 is freed. + // Blocks 4 and 5 are shared with the clone and the snapshot; only the new block 6 is freed. before := freeBlocks() if err := os.Truncate(path, 4*mb); err != nil { t.Fatalf("Truncate failed: %v", err) @@ -264,9 +261,10 @@ func TestExtentTreeSnapshot(t *testing.T) { if err := unix.Removexattr(path, "user.snapshot.tree-snap"); err != nil && err != syscall.ENODATA { t.Fatalf("deleting snapshot failed: %v", err) } - time.Sleep(500 * time.Millisecond) - if got := freeBlocks() - before; got != 0 { - t.Errorf("deleting the snapshot freed %d blocks, want 0", got) + waitForSnapshotState(t, path, "", "tree-snap") + // The snapshot's own extent tree node is the only block nothing else maps. + if got := freeBlocks() - before; got != 1 { + t.Errorf("deleting the snapshot freed %d blocks, want 1", got) } remount(t, mfs) diff --git a/tests/integration/snapshot_test.go b/tests/integration/snapshot_test.go index 2ec912e4..b05459af 100644 --- a/tests/integration/snapshot_test.go +++ b/tests/integration/snapshot_test.go @@ -3,8 +3,10 @@ package integration import ( "bytes" + "fmt" "os" "strings" + "sync" "syscall" "testing" "time" @@ -504,9 +506,7 @@ func TestSnapshotDeleteAndReuse(t *testing.T) { if err != nil && err != syscall.ENODATA { t.Fatalf("Removexattr failed: %v", err) } - - // Wait for background deletion and block freeing - time.Sleep(500 * time.Millisecond) + waitForSnapshotState(t, filePath, "", "temp-snap") // Verify the live file still has the new content readBack, err := mfs.ReadFile("reuse.txt") @@ -823,3 +823,193 @@ func TestCloneFromSnapshot(t *testing.T) { t.Logf("Clone content: %q", cloneContent) t.Log("Clone from snapshot test passed") } + +func setxattr(t *testing.T, path, attr, value string) { + t.Helper() + if err := unix.Setxattr(path, attr, []byte(value), 0); err != nil { + t.Fatalf("setxattr %s=%q on %s: %v", attr, value, path, err) + } +} + +func overwrite(base []byte, off int, data []byte) []byte { + out := bytes.Clone(base) + copy(out[off:], data) + return out +} + +func freeBlocks(t *testing.T, mfs *MountedFS) uint64 { + t.Helper() + var st unix.Statfs_t + if err := unix.Statfs(mfs.Path(""), &st); err != nil { + t.Fatalf("statfs: %v", err) + } + return st.Bfree +} + +// TestSnapshotPreservesData reads a snapshot, through a clone of it, after the +// file was overwritten in place, appended to and truncated. +func TestSnapshotPreservesData(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + original := chunk(1, 3*mb) + if err := writer.WriteFile("data.bin", original); err != nil { + t.Fatal(err) + } + checkFile(t, writer, "data.bin", original) + setxattr(t, writer.Path("data.bin"), "user.snapshot", "before") + + if err := writer.WriteFileAt("data.bin", chunk(2, 4096), mb+100); err != nil { + t.Fatal(err) + } + if err := writer.WriteFileAt("data.bin", chunk(3, 2*mb), 3*mb); err != nil { + t.Fatal(err) + } + if err := os.Truncate(writer.Path("data.bin"), mb+mb/2); err != nil { + t.Fatal(err) + } + setxattr(t, writer.Path("data.bin"), "user.snapshot.clone", "before:/before-clone") + + live := overwrite(original[:mb+mb/2], mb+100, chunk(2, 4096)) + for _, node := range []*MountedFS{writer, reader} { + checkFile(t, node, "before-clone", original) + checkFile(t, node, "data.bin", live) + } +} + +// TestCloneInPlaceWrite writes into a clone and its source without truncating, +// after both were read, so the reads that follow must not reuse cached blocks. +func TestCloneInPlaceWrite(t *testing.T) { + mfs := NewMountedFS(t, 20) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + original := chunk(4, 2*mb) + if err := mfs.WriteFile("source.bin", original); err != nil { + t.Fatal(err) + } + setxattr(t, mfs.Path("source.bin"), "user.clone", "/clone.bin") + checkFile(t, mfs, "source.bin", original) + checkFile(t, mfs, "clone.bin", original) + + if err := mfs.WriteFileAt("clone.bin", chunk(5, 100), 10); err != nil { + t.Fatal(err) + } + if err := mfs.WriteFileAt("source.bin", chunk(6, 100), mb+10); err != nil { + t.Fatal(err) + } + + wantSource := overwrite(original, mb+10, chunk(6, 100)) + wantClone := overwrite(original, 10, chunk(5, 100)) + checkFile(t, mfs, "source.bin", wantSource) + checkFile(t, mfs, "clone.bin", wantClone) + remount(t, mfs) + checkFile(t, mfs, "source.bin", wantSource) + checkFile(t, mfs, "clone.bin", wantClone) +} + +// TestSnapshotDeleteAfterOverwrite deletes a snapshot whose blocks the file no +// longer maps; the delete must finish and give back exactly those blocks. +func TestSnapshotDeleteAfterOverwrite(t *testing.T) { + mfs := NewMountedFS(t, 20) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + if err := mfs.WriteFile("data.bin", chunk(7, 2*mb)); err != nil { + t.Fatal(err) + } + free := freeBlocks(t, mfs) + setxattr(t, mfs.Path("data.bin"), "user.snapshot", "temp") + + updated := chunk(8, 2*mb) + if err := mfs.WriteFileAt("data.bin", updated, 0); err != nil { + t.Fatal(err) + } + if got := freeBlocks(t, mfs); got != free-2 { + t.Fatalf("free blocks after overwriting 2 snapshotted blocks = %d, want %d", got, free-2) + } + + if err := unix.Removexattr(mfs.Path("data.bin"), "user.snapshot.temp"); err != nil { + t.Fatalf("delete snapshot: %v", err) + } + waitForSnapshotState(t, mfs.Path("data.bin"), "", "temp") + if got := freeBlocks(t, mfs); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, mfs, "data.bin", updated) +} + +// TestSnapshotConcurrentCreatesAcrossNodes has two nodes snapshot their own +// files at the same time. Each file keeps all its snapshots, and deleting one +// node's snapshots leaves the other node's alone. +func TestSnapshotConcurrentCreatesAcrossNodes(t *testing.T) { + if testing.Short() { + t.Skip("two nodes can still both claim a free inode until lock claims are confirmed (delta-lease)") + } + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + nodes := []*MountedFS{cluster.Node(0), cluster.Node(1)} + const perNode = 10 + file := func(n int) string { return fmt.Sprintf("conc%d.bin", n) } + snapName := func(n, i int) string { return fmt.Sprintf("conc%d-%d", n, i) } + for n, node := range nodes { + if err := node.WriteFile(file(n), chunk(byte(n+1), mb)); err != nil { + t.Fatal(err) + } + } + + var wg sync.WaitGroup + errs := make(chan error, 2*perNode) + for n, node := range nodes { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perNode { + if err := unix.Setxattr(node.Path(file(n)), "user.snapshot", []byte(snapName(n, i)), 0); err != nil { + errs <- fmt.Errorf("node %d snapshot %d: %w", n, i, err) + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + names := func(n int) []string { + var out []string + for i := range perNode { + out = append(out, snapName(n, i)) + } + return out + } + for n, node := range nodes { + waitForSnapshotState(t, node.Path(file(n)), "active", names(n)...) + } + + for _, name := range names(0) { + if err := unix.Removexattr(nodes[0].Path(file(0)), "user.snapshot."+name); err != nil { + t.Fatalf("delete %s: %v", name, err) + } + } + waitForSnapshotState(t, nodes[0].Path(file(0)), "", names(0)...) + if states := snapshotStates(t, nodes[1].Path(file(1))); len(states) != perNode { + t.Fatalf("node 1's file lists %d snapshots after node 0 deleted its own, want %d: %v", len(states), perNode, states) + } + setxattr(t, nodes[1].Path(file(1)), "user.snapshot.clone", snapName(1, 0)+":/conc1-clone.bin") + checkFile(t, nodes[1], "conc1-clone.bin", chunk(2, mb)) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index eb9c88af..499c4658 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -650,7 +650,7 @@ scenarios: - id: extent-tree-snapshot name: Snapshot and clone of a fragmented file - description: A clone of a file with more than 2 extents gets its own extent tree; blocks it shares survive truncating and deleting the source, and deleting the snapshot frees nothing in use + description: A clone of a file with more than 2 extents gets its own extent tree; blocks it shares survive truncating and deleting the source, and deleting the snapshot frees only its own extent tree node priority: high # Docker/VM TODO: snapshot and clone tests need xattr support in those harnesses, like the other snapshot scenarios. environments: @@ -1287,20 +1287,146 @@ scenarios: environments: go: internal/snapshot/worker_test.go::TestWorkerStartStop - - id: snapshot-worker-create-job - name: Snapshot create job processing - description: Background worker processes refcount increment jobs + - id: snapshot-worker-recovery + name: Snapshot recovery after a crash + description: On mount, snapshots left creating or deleting give back their captured blocks and disappear, without touching blocks the live file still maps priority: go_only environments: - go: internal/snapshot/worker_test.go::TestWorkerQueueCreateJob + go: internal/snapshot/worker_test.go::TestRecoverIncompleteSnapshots - id: snapshot-worker-delete-job name: Snapshot delete job processing - description: Background worker processes refcount decrement jobs + description: Background worker frees the snapshot's own blocks, keeps blocks the live file still maps, and removes the descriptor priority: go_only environments: go: internal/snapshot/worker_test.go::TestWorkerQueueDeleteJob + - id: snapshot-queue-canceled-context + name: Snapshot delete queued from a canceled request + description: A delete job queued with an already canceled request context still runs + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestWorkerQueuesJobsWithCanceledContext + + - id: refcount-share-release + name: Block reference sharing + description: Sharing a block counts the existing owner, and releasing reports a block as free only when its last owner lets go + priority: go_only + environments: + go: internal/refcount/refcount_test.go::TestShareAndRelease + + - id: extent-free-shared-blocks + name: Freeing shared blocks + description: Truncating one of two files that share a block keeps the block allocated until the other file lets go + priority: go_only + environments: + go: internal/extent/extent_test.go::TestFreeKeepsSharedBlocks + + - id: snapshot-refuses-uncounted-sharing + name: Snapshot of a file shared by an older version + description: Snapshotting or cloning a file whose blocks an older version shared without refcounts fails with ENOTSUP and leaves nothing behind + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRefusesUncountedSharing + + - id: extent-uncounted-sharing-never-freed + name: Blocks shared by an older version stay allocated + description: Truncating a file that shares blocks without refcounts, with a count of 0 or 1, never frees them + priority: go_only + environments: + go: internal/extent/extent_test.go::TestUncountedSharingIsNeverFreed + + - id: extent-concurrent-inline-cow + name: Two nodes copy the same shared block + description: When two copies of an inline inode copy the same shared block, the second inode write fails with a tree conflict and the old block is released once + priority: go_only + environments: + go: internal/extent/extent_test.go::TestConcurrentInlineCopyOnWrite + + - id: snapshot-concurrent-cow-across-nodes + name: Two nodes copy one shared block through the write path + description: Node A commits its copy of a shared block first; node B's commit fails, B frees only its own copy and retries in place, and the old block loses exactly one owner + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestConcurrentCopyOnWriteAcrossNodes + + - id: snapshot-old-image-safe + name: Old snapshot and old clone after upgrading + description: On an image with an older version's snapshot and clone of one file, writes and deleting the snapshot, clone and source never free a block another inode maps + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestOldSnapshotAndCloneNeverFreeSharedBlocks + + - id: snapshot-old-sharing-cow + name: Copy-on-write of blocks an older version shared + description: Writing a file and one of its clones from an older version copies the uncounted block but never releases it, so the other clone keeps its data + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestOldSharingCopiesWithoutReleasing + + - id: snapshot-delete-job-requires-deleting + name: Delete job only for deleting snapshots + description: A delete job for an active snapshot fails and leaves the snapshot, its data inode and refcounts alone + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState + + - id: snapshot-failed-write-releases-once + name: Failed write after a mid-write copy commit + description: When the final inode write fails after a mid-write persist committed one copied block, each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestFailedWriteReleasesCommittedCopiesOnce + + - id: snapshot-stale-copy-after-cow + name: Stale inode copy after a copy-on-write + description: An inode copy read before a copy-on-write committed cannot write the old mapping back; a real chmod retries and keeps the copy + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestStaleCopyCannotUndoCopyOnWrite + + - id: inode-stale-inline-copy + name: Stale inline inode copy + description: Writing an inode copy whose inline extents changed on disk since it was read fails with ErrTreeConflict + priority: go_only + environments: + go: internal/inode/inode_test.go::TestStaleCopyCannotUndoInlineExtentChange + + - id: snapshot-recovery-after-crash + name: Snapshot cleanup on mount after a crash + description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRecoveryAfterCrash + + - id: snapshot-ids-unique-across-nodes + name: Snapshot IDs across nodes + description: Two mounted nodes creating snapshots get different IDs + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryIDsUniqueAcrossNodes + + - id: snapshot-ids-after-older-images + name: Snapshot IDs on an older image and after slot reuse + description: New IDs land above the sequential IDs an older version assigned, and a node that takes over a departed node's slot never reuses its IDs + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryIDsAfterOlderImagesAndSlotReuse + + - id: snapshot-activate-requires-creating + name: Snapshot activation after a rollback + description: A snapshot another node rolled back cannot be marked active + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryActivateRequiresCreating + + - id: snapshot-cow-in-process + name: Snapshot and clone copy-on-write + description: Snapshots keep their data through in-place writes, appends and truncates; clone writes never reach the source; deleting a snapshot after an overwrite frees exactly its blocks + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotKeepsDataThroughWrites + # Task #4712: Snapshot deletion (leaf + merge-up) - id: snapshot-leaf-deletion name: Leaf snapshot deletion @@ -1397,6 +1523,49 @@ scenarios: docker: TODO vm: TODO + - id: snapshot-preserves-data + name: Snapshot keeps its data + description: After a snapshot, in-place overwrite, append and truncate of the file; a clone of the snapshot read on another node holds the original data + priority: critical + environments: + go: snapshot_test.go::TestSnapshotPreservesData + docker: run-tests.sh::test_snapshot_preserves_data + vm: run-vm-tests.sh::test_snapshot_preserves_data + + - id: clone-in-place-write + name: In-place write to a clone + description: dd conv=notrunc into a clone, and into its source, changes only the file written + priority: critical + environments: + go: snapshot_test.go::TestCloneInPlaceWrite + docker: run-tests.sh::test_clone_in_place_write + vm: run-vm-tests.sh::test_clone_in_place_write + + - id: snapshot-delete-after-overwrite + name: Snapshot delete after overwrite + description: Deleting a snapshot after the file was overwritten finishes, frees the snapshot's blocks, and leaves the file intact + priority: high + environments: + go: snapshot_test.go::TestSnapshotDeleteAfterOverwrite + docker: run-tests.sh::test_snapshot_delete_after_overwrite + vm: run-vm-tests.sh::test_snapshot_delete_after_overwrite + + - id: snapshot-concurrent-creates-across-nodes + name: Concurrent snapshots on two nodes + description: Two nodes snapshot their own files at the same time; each file keeps all its snapshots, and deleting one node's snapshots leaves the other's + priority: high + environments: + go: snapshot_test.go::TestSnapshotConcurrentCreatesAcrossNodes + docker: run-tests.sh::test_snapshot_concurrent_creates + vm: run-vm-tests.sh::test_snapshot_concurrent_creates + + - id: snapshot-registry-concurrent-creates + name: Registry creates from two nodes at once + description: Two registries create snapshots concurrently with no lock between them; node slots keep every slot and ID apart + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryConcurrentCreatesAcrossNodes + - id: clone-from-snapshot name: Clone from snapshot description: Create writable clone from named snapshot via xattr diff --git a/tests/vm/scripts/build-vm-image.sh b/tests/vm/scripts/build-vm-image.sh index b66249be..f7f4c898 100755 --- a/tests/vm/scripts/build-vm-image.sh +++ b/tests/vm/scripts/build-vm-image.sh @@ -121,7 +121,7 @@ EOF log_info "Installing packages..." sudo chroot "${MOUNT_POINT}" /bin/sh -c " apk update - apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs + apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs attr # Configure SSH ssh-keygen -A diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 25df7901..28f38e64 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2860,6 +2860,190 @@ test_extent_tree_sparse_writes() { return 0 } +md5_on_vm_matches() { + local vm_id=$1 path=$2 want=$3 got + got=$(ssh_run "$vm_id" "md5sum < $path | cut -d' ' -f1") + if [ "$got" != "$want" ]; then + log_error "VM $vm_id $path md5 $got, want $want" + return 1 + fi +} + +# Snapshots a file, then overwrites it in place, appends to it and truncates +# it. A clone of the snapshot must hold the original data on both VMs. The +# expected data is built with the same edits on a local copy. +test_snapshot_preserves_data() { + log_info "Testing that a snapshot keeps its data..." + ensure_cluster_healthy 2 + + local file=/mnt/goclusterfs/snap_data.bin + local clone=/mnt/goclusterfs/snap_data_clone.bin + + ssh_run 1 "rm -f $file $clone" + ssh_run 1 "head -c 3145728 /dev/urandom > /tmp/snap_orig && cp /tmp/snap_orig /tmp/snap_live && cp /tmp/snap_orig $file" || return 1 + ssh_run 1 "setfattr -n user.snapshot -v snap_data_before $file" || { + log_error "snapshot failed" + return 1 + } + ssh_run 1 "head -c 4096 /dev/urandom > /tmp/snap_patch && head -c 2097152 /dev/urandom > /tmp/snap_tail" + for target in "$file" /tmp/snap_live; do + ssh_run 1 "dd if=/tmp/snap_patch of=$target bs=4096 seek=257 conv=notrunc 2>/dev/null && cat /tmp/snap_tail >> $target && truncate -s 1572864 $target" || { + log_error "changing $target failed" + return 1 + } + done + ssh_run 1 "setfattr -n user.snapshot.clone -v snap_data_before:/snap_data_clone.bin $file" || { + log_error "clone from snapshot failed" + return 1 + } + + local want_clone want_live + want_clone=$(ssh_run 1 "md5sum < /tmp/snap_orig | cut -d' ' -f1") + want_live=$(ssh_run 1 "md5sum < /tmp/snap_live | cut -d' ' -f1") + for vm_id in 2 1; do + restart_goclusterfs_vm "$vm_id" || return 1 + md5_on_vm_matches "$vm_id" "$clone" "$want_clone" || return 1 + md5_on_vm_matches "$vm_id" "$file" "$want_live" || return 1 + done + + ssh_run 1 "setfattr -x user.snapshot.snap_data_before $file" + log_info "Snapshot keeps its data test passed" + return 0 +} + +# Writes into a clone and into its source with dd conv=notrunc. Each write must +# change only the file written, on both VMs. +test_clone_in_place_write() { + log_info "Testing in-place writes to a clone and its source..." + ensure_cluster_healthy 2 + + local src=/mnt/goclusterfs/clone_src.bin + local clone=/mnt/goclusterfs/clone_inplace.bin + + ssh_run 1 "rm -f $src $clone" + ssh_run 1 "head -c 2097152 /dev/urandom > /tmp/clone_src && cp /tmp/clone_src /tmp/clone_copy && cp /tmp/clone_src $src" || return 1 + ssh_run 1 "setfattr -n user.clone -v /clone_inplace.bin $src" || { + log_error "clone failed" + return 1 + } + ssh_run 1 "head -c 4096 /dev/urandom > /tmp/clone_patch_a && head -c 4096 /dev/urandom > /tmp/clone_patch_b" + ssh_run 1 "dd if=/tmp/clone_patch_a of=$clone bs=4096 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_a of=/tmp/clone_copy bs=4096 conv=notrunc 2>/dev/null" || return 1 + ssh_run 1 "dd if=/tmp/clone_patch_b of=$src bs=4096 seek=256 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_b of=/tmp/clone_src bs=4096 seek=256 conv=notrunc 2>/dev/null" || return 1 + + local want_src want_clone + want_src=$(ssh_run 1 "md5sum < /tmp/clone_src | cut -d' ' -f1") + want_clone=$(ssh_run 1 "md5sum < /tmp/clone_copy | cut -d' ' -f1") + for vm_id in 2 1; do + restart_goclusterfs_vm "$vm_id" || return 1 + md5_on_vm_matches "$vm_id" "$src" "$want_src" || return 1 + md5_on_vm_matches "$vm_id" "$clone" "$want_clone" || return 1 + done + + log_info "In-place clone write test passed" + return 0 +} + +# Snapshots a file, overwrites both of its blocks, then deletes the snapshot. +# The delete must finish and give back exactly the two blocks only the +# snapshot held. +test_snapshot_delete_after_overwrite() { + log_info "Testing snapshot delete after overwrite..." + ensure_cluster_healthy 2 + + local file=/mnt/goclusterfs/snap_delete.bin + local free_before free_now + + ssh_run 1 "rm -f $file" + ssh_run 1 "head -c 2097152 /dev/urandom > $file" || return 1 + free_before=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + ssh_run 1 "setfattr -n user.snapshot -v snap_delete_temp $file" || { + log_error "snapshot failed" + return 1 + } + ssh_run 1 "head -c 2097152 /dev/urandom > /tmp/snap_new && dd if=/tmp/snap_new of=$file bs=1048576 conv=notrunc 2>/dev/null" || return 1 + free_now=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + if [ "$free_now" -ne $((free_before - 2)) ]; then + log_error "$free_now free blocks after overwriting 2 snapshotted blocks, want $((free_before - 2))" + return 1 + fi + + ssh_run 1 "setfattr -x user.snapshot.snap_delete_temp $file" || { + log_error "snapshot delete failed" + return 1 + } + local tries=0 + while ssh_run 1 "getfattr --only-values -n user.snapshots $file 2>/dev/null | grep -q snap_delete_temp"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log_error "snapshot still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + free_now=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + if [ "$free_now" -ne "$free_before" ]; then + log_error "$free_now free blocks after deleting the snapshot, want $free_before" + return 1 + fi + md5_on_vm_matches 1 "$file" "$(ssh_run 1 "md5sum < /tmp/snap_new | cut -d' ' -f1")" || return 1 + + log_info "Snapshot delete after overwrite test passed" + return 0 +} + +# Two VMs each take 10 snapshots of their own file at the same time. Each file +# must list all of its snapshots, and deleting one VM's snapshots must leave +# the other VM's alone. +test_snapshot_concurrent_creates() { + log_info "Testing concurrent snapshot creation on two VMs..." + ensure_cluster_healthy 2 + + local run=$$ + local file_a=/mnt/goclusterfs/conc_a.bin + local file_b=/mnt/goclusterfs/conc_b.bin + + ssh_run 1 "rm -f $file_a && head -c 1048576 /dev/urandom > $file_a" || return 1 + ssh_run 2 "rm -f $file_b && head -c 1048576 /dev/urandom > $file_b" || return 1 + + ssh_run 1 "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_a_${run}_\$i $file_a || exit 1; done" & + local pid_a=$! + ssh_run 2 "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_b_${run}_\$i $file_b || exit 1; done" & + local pid_b=$! + wait "$pid_a" || { log_error "snapshots on VM 1 failed"; return 1; } + wait "$pid_b" || { log_error "snapshots on VM 2 failed"; return 1; } + + local count_a count_b + count_a=$(ssh_run 1 "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -c ':active'") + count_b=$(ssh_run 2 "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_a" -ne 10 ] || [ "$count_b" -ne 10 ]; then + log_error "$count_a and $count_b active snapshots, want 10 each" + return 1 + fi + + ssh_run 1 "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_a_${run}_\$i $file_a || exit 1; done" || { + log_error "deleting snapshots on VM 1 failed" + return 1 + } + local tries=0 + while ssh_run 1 "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -q conc_a_"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log_error "VM 1 snapshots still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + count_b=$(ssh_run 2 "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_b" -ne 10 ]; then + log_error "VM 2 lists $count_b snapshots after VM 1 deleted its own, want 10" + return 1 + fi + + ssh_run 2 "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_b_${run}_\$i $file_b; done" + log_info "Concurrent snapshot creation test passed" + return 0 +} + # Two VMs append 1MB alternately to their own file, so both files get # non-contiguous blocks and grow past the old 2MB limit. Checks both files on # each VM after restarting it. @@ -2961,6 +3145,10 @@ main() { run_test "Sequential Access" test_sequential_access run_test "Extent Tree Sparse Writes" test_extent_tree_sparse_writes run_test "Extent Tree Interleaved Appends" test_extent_tree_interleaved_appends + run_test "Snapshot Preserves Data" test_snapshot_preserves_data + run_test "Clone In-Place Write" test_clone_in_place_write + run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite + run_test "Snapshot Concurrent Creates" test_snapshot_concurrent_creates run_test "Concurrent Writes" test_concurrent_writes run_test "Concurrent File Creation" test_concurrent_file_creation diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 34d68c4e..b7c26bd1 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -361,9 +361,9 @@ setup_goclusterfs() { "${PROJECT_ROOT}/goclusterfs-mount" \ "root@127.0.0.1:/usr/local/bin/goclusterfs-mount" - # Install fuse3 and gcompat (glibc compatibility for binary compiled on glibc systems) - log_info "Installing fuse3 and gcompat..." - ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat" + # Install fuse3, gcompat (glibc compatibility for binary compiled on glibc systems) and attr (setfattr for snapshot tests) + log_info "Installing fuse3, gcompat and attr..." + ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat attr" # Load FUSE kernel module (required before mounting any FUSE filesystem) log_info "Loading FUSE kernel module..."