From 33fac6cf1d59cc39d43c0274d3caf4c0a8b0e879 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:26:46 -0500 Subject: [PATCH 01/18] fix(refcount): count the existing owner when a block is first shared New blocks start at a count of 0, so a snapshot raised them to 1 and copy-on-write, which needs more than 1, never fired. Share treats 0 and 1 alike as one owner and adds the new one, and Release drops an owner and reports the blocks nobody holds any more. Co-Authored-By: Claude Opus 5.5 --- internal/refcount/refcount.go | 179 ++++++++++++++++------------- internal/refcount/refcount_test.go | 55 +++++++++ 2 files changed, 153 insertions(+), 81 deletions(-) diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 6863179c..4ada00f8 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -1,11 +1,13 @@ // Package refcount implements block reference counting for GoClusterFS snapshots. // // Reference counting enables Copy-on-Write (CoW) semantics for snapshots: -// - RefCount = 1: Block is exclusive to one file, can modify in place -// - RefCount > 1: Block is shared (via snapshot), must CoW before writing +// - RefCount = 0 or 1: Block has one owner and can be modified in place. +// New blocks start at 0, so allocation never touches this region. +// - RefCount > 1: Block is shared by that many owners (files, clones, +// snapshots) and must be copied before writing. // -// This allows O(1) snapshot creation - just increment refcounts of all blocks. -// When a write targets a shared block, allocate a new block and copy data. +// Share adds an owner and Release drops one; a block goes back to the bitmap +// only when Release reports that its last owner let go. // // Layout: Each entry is 8 bytes (4-byte count + 2-byte flags + 2-byte reserved). // Entries are stored in 4KB sectors with a CRC32C checksum at the end. @@ -272,82 +274,104 @@ func (r *Region) SetFlags(ctx context.Context, blockNum uint64, flags EntryFlags // This is more efficient than calling Increment individually for each block. // Returns the new counts after incrementing. func (r *Region) IncrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { - if len(blockNums) == 0 { - return nil, nil - } - - for _, blockNum := range blockNums { - if blockNum >= r.totalBlocks { - return nil, ErrBlockOutOfRange + results := make([]uint32, len(blockNums)) + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount == 0xFFFFFFFF { + return false, ErrRefCountOverflow } + e.RefCount++ + results[i] = e.RefCount + return true, nil + }) + if err != nil || len(blockNums) == 0 { + return nil, err } + return results, nil +} - // Acquire distributed lock if available - if r.lockMgr != nil { - if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire refcount lock: %w", err) +// DecrementBatch decrements reference counts for multiple blocks atomically. +// Returns the new counts after decrementing. +// Returns ErrRefCountUnderflow if any block's count is already zero. +func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { + results := make([]uint32, len(blockNums)) + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount == 0 { + return false, ErrRefCountUnderflow } - defer r.lockMgr.Release(lock.LockTypeRefCount, 0) + e.RefCount-- + results[i] = e.RefCount + return true, nil + }) + if err != nil || len(blockNums) == 0 { + return nil, err } + return results, nil +} - r.mu.Lock() - defer r.mu.Unlock() - - results := make([]uint32, len(blockNums)) - - // Group blocks by sector for efficient I/O - sectorCache := make(map[uint64][]byte) - - for i, blockNum := range blockNums { - sectorIdx := blockNum / EntriesPerSector - - // Load sector if not cached - if _, ok := sectorCache[sectorIdx]; !ok { - sectorData, err := r.readSector(sectorIdx) - if err != nil { - return nil, err - } - sectorCache[sectorIdx] = sectorData +// Share adds one owner, such as a snapshot or a clone, to each block. Blocks +// are allocated with a count of 0, which like 1 means a single owner, so the +// first share of a block sets its count to 2. +func (r *Region) Share(ctx context.Context, blockNums []uint64) error { + return r.updateBatch(ctx, blockNums, func(_ int, e *Entry) (bool, error) { + if e.RefCount == 0xFFFFFFFF { + return false, ErrRefCountOverflow } + e.RefCount = max(e.RefCount, 1) + 1 + return true, nil + }) +} - // Read and increment entry - entry := decodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector) - if entry.RefCount == 0xFFFFFFFF { - return nil, ErrRefCountOverflow +// Release drops one owner from each block and returns the blocks that no +// owner holds any more. The caller must free those in the bitmap. +func (r *Region) Release(ctx context.Context, blockNums []uint64) ([]uint64, error) { + var free []uint64 + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + if e.RefCount > 1 { + e.RefCount-- + return true, nil } - entry.RefCount++ - results[i] = entry.RefCount - encodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector, entry) - } - - // Write back modified sectors - for sectorIdx, sectorData := range sectorCache { - if err := r.writeSector(sectorIdx, sectorData); err != nil { - return nil, err + free = append(free, blockNums[i]) + if *e == (Entry{}) { + return false, nil } + *e = Entry{} + return true, nil + }) + if err != nil { + return nil, err } + return free, nil +} - return results, nil +// Counts returns the reference count of each block. +func (r *Region) Counts(ctx context.Context, blockNums []uint64) ([]uint32, error) { + counts := make([]uint32, len(blockNums)) + err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { + counts[i] = e.RefCount + return false, nil + }) + if err != nil { + return nil, err + } + return counts, nil } -// DecrementBatch decrements reference counts for multiple blocks atomically. -// Returns the new counts after decrementing. -// Returns ErrRefCountUnderflow if any block's count is already zero. -func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint32, error) { +// updateBatch applies fn to the entry of each block under the region lock and +// writes back the sectors fn changed. Nothing is written if fn fails. +func (r *Region) updateBatch(ctx context.Context, blockNums []uint64, fn func(i int, e *Entry) (bool, error)) error { if len(blockNums) == 0 { - return nil, nil + return nil } - for _, blockNum := range blockNums { if blockNum >= r.totalBlocks { - return nil, ErrBlockOutOfRange + return ErrBlockOutOfRange } } // Acquire distributed lock if available if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { - return nil, fmt.Errorf("failed to acquire refcount lock: %w", err) + return fmt.Errorf("failed to acquire refcount lock: %w", err) } defer r.lockMgr.Release(lock.LockTypeRefCount, 0) } @@ -355,41 +379,34 @@ func (r *Region) DecrementBatch(ctx context.Context, blockNums []uint64) ([]uint r.mu.Lock() defer r.mu.Unlock() - results := make([]uint32, len(blockNums)) - - // Group blocks by sector for efficient I/O - sectorCache := make(map[uint64][]byte) - + sectors := make(map[uint64][]byte) + dirty := make(map[uint64]bool) for i, blockNum := range blockNums { sectorIdx := blockNum / EntriesPerSector - - // Load sector if not cached - if _, ok := sectorCache[sectorIdx]; !ok { - sectorData, err := r.readSector(sectorIdx) + if _, ok := sectors[sectorIdx]; !ok { + data, err := r.readSector(sectorIdx) if err != nil { - return nil, err + return err } - sectorCache[sectorIdx] = sectorData + sectors[sectorIdx] = data } - - // Read and decrement entry - entry := decodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector) - if entry.RefCount == 0 { - return nil, ErrRefCountUnderflow + entry := decodeEntry(sectors[sectorIdx], blockNum%EntriesPerSector) + changed, err := fn(i, &entry) + if err != nil { + return err + } + if changed { + encodeEntry(sectors[sectorIdx], blockNum%EntriesPerSector, entry) + dirty[sectorIdx] = true } - entry.RefCount-- - results[i] = entry.RefCount - encodeEntry(sectorCache[sectorIdx], blockNum%EntriesPerSector, entry) } - // Write back modified sectors - for sectorIdx, sectorData := range sectorCache { - if err := r.writeSector(sectorIdx, sectorData); err != nil { - return nil, err + for sectorIdx := range dirty { + if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil { + return err } } - - return results, nil + return nil } // Scan reads all reference counts and returns statistics. diff --git a/internal/refcount/refcount_test.go b/internal/refcount/refcount_test.go index beb18bc8..a05f8123 100644 --- a/internal/refcount/refcount_test.go +++ b/internal/refcount/refcount_test.go @@ -566,6 +566,61 @@ func TestDecrementBatchUnderflow(t *testing.T) { } } +func TestShareAndRelease(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + region := NewRegion(dev, 0, 1000, nil) + ctx := context.Background() + blocks := []uint64{3, 700} + count := func(block uint64) uint32 { + t.Helper() + e, err := region.Get(ctx, block) + if err != nil { + t.Fatal(err) + } + return e.RefCount + } + + if err := region.Share(ctx, blocks); err != nil { + t.Fatal(err) + } + if got := count(3); got != 2 { + t.Fatalf("first share of a new block: count %d, want 2", got) + } + if err := region.Share(ctx, blocks[:1]); err != nil { + t.Fatal(err) + } + if got := count(3); got != 3 { + t.Fatalf("second share: count %d, want 3", got) + } + + for want := uint32(2); want >= 1; want-- { + free, err := region.Release(ctx, blocks[:1]) + if err != nil { + t.Fatal(err) + } + if len(free) != 0 || count(3) != want { + t.Fatalf("release down to %d: free=%v count=%d", want, free, count(3)) + } + } + free, err := region.Release(ctx, blocks) + if err != nil { + t.Fatal(err) + } + if len(free) != 1 || free[0] != 3 || count(3) != 0 || count(700) != 1 { + t.Fatalf("last owner of 3 and one of two owners of 700: free=%v counts=%d,%d", free, count(3), count(700)) + } + + free, err = region.Release(ctx, []uint64{700, 900}) + if err != nil { + t.Fatal(err) + } + if len(free) != 2 || count(700) != 0 || count(900) != 0 { + t.Fatalf("release of single-owner blocks: free=%v", free) + } +} + func TestScan(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() From e91eab419bf5d49df1f7dab36e8bbd49d8432a8f Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:26:46 -0500 Subject: [PATCH 02/18] fix(extent): free shared blocks only after their last owner lets go Freed blocks now drop one owner in the refcount table and go back to the bitmap only when none is left. Shared blocks used to be skipped and leaked. A shared block with a count of 0 comes from a clone made by an older version, so it stays allocated. A copy-on-write remap bumps DataGeneration so readers drop cached block mappings. Co-Authored-By: Claude Opus 5.5 --- internal/extent/extent.go | 53 ++++++++++++++++++++++++++----- internal/extent/extent_test.go | 58 ++++++++++++++++++++++++++++++++++ internal/extent/tree_edit.go | 36 +++++++++++++++++---- 3 files changed, 133 insertions(+), 14 deletions(-) diff --git a/internal/extent/extent.go b/internal/extent/extent.go index 6ea3fab0..a06d41b8 100644 --- a/internal/extent/extent.go +++ b/internal/extent/extent.go @@ -86,10 +86,18 @@ type Manager struct { leafCap int // extents per leaf before it splits indexCap int // leaves per tree + refs *refcount.Region // nil when snapshots are disabled + deferredMu sync.Mutex deferred []uint64 // frees postponed by lock order } +// SetRefcounts makes frees respect block sharing: a block shared by a +// snapshot or clone goes back to the bitmap only after its last owner frees it. +func (m *Manager) SetRefcounts(refs *refcount.Region) { + m.refs = refs +} + // NewManager creates a new extent manager. dataOffset is the device offset of // data block 0; extent tree nodes are stored in data blocks. func NewManager(dev *storage.Device, bm *bitmap.Region, lockMgr *lock.Manager, dataOffset uint64) *Manager { @@ -315,9 +323,8 @@ func (m *Manager) unmap(ctx context.Context, ino *inode.Inode, from, to uint64) // release updates block accounting for removed extents and queues their // blocks to be freed when the inode write commits: if that write loses a -// conflict, the inode on disk still maps them. Shared blocks stay allocated for -// the refcount system, and SFB extents point into containers rather than at -// bitmap blocks. +// conflict, the inode on disk still maps them. SFB extents point into +// containers rather than at bitmap blocks. func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Extent) { var blocks []uint64 for _, x := range removed { @@ -325,11 +332,8 @@ func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Exten if x.Flags&FlagUnwritten == 0 { ino.WrittenBlocks -= min(ino.WrittenBlocks, x.Length) } - if x.Flags&(FlagShared|FlagSFB) != 0 { - continue - } - for j := uint64(0); j < x.Length; j++ { - blocks = append(blocks, x.PhysicalBlock+j) + if x.Flags&FlagSFB == 0 { + blocks = append(blocks, m.ownedBlocks(ctx, x)...) } } if len(blocks) == 0 { @@ -343,6 +347,37 @@ func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Exten p.release = append(p.release, blocks...) } +// ownedBlocks returns the blocks of x that refcounts can account for. A +// shared block with a count of 0 comes from a clone made by an older version, +// which shared blocks without counting the owners, so it stays allocated. So +// do all shared blocks when refcounts are disabled. +func (m *Manager) ownedBlocks(ctx context.Context, x Extent) []uint64 { + blocks := make([]uint64, x.Length) + for j := range blocks { + blocks[j] = x.PhysicalBlock + uint64(j) + } + if x.Flags&FlagShared == 0 { + return blocks + } + if m.refs == nil { + return nil + } + counts, err := m.refs.Counts(ctx, blocks) + if err != nil { + log.Printf("[extent] read refcounts of blocks %d-%d: %v; leaving them allocated", x.PhysicalBlock, x.PhysicalBlock+x.Length-1, err) + return nil + } + owned := blocks[:0] + for j, b := range blocks { + if counts[j] == 0 { + log.Printf("[extent] block %d is shared but uncounted; leaving it allocated", b) + continue + } + owned = append(owned, b) + } + return owned +} + // Delete unmaps every block of an inode that is being freed and frees the // blocks now, since no inode write will follow to commit the change. func (m *Manager) Delete(ctx context.Context, ino *inode.Inode) error { @@ -570,6 +605,8 @@ func (m *Manager) CopyOnWrite(ctx context.Context, ino *inode.Inode, ext *Extent return nil, err } *ext = private + // Readers keep cached block mappings until DataGeneration changes. + ino.DataGeneration++ result.NewPhysBlock = newPhysBlock result.WasShared = true diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index 2f1f5040..bfeac5bc 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -1507,3 +1507,61 @@ func listExtents(tb testing.TB, mgr *Manager, ino *inode.Inode) []*Extent { } return extents } + +func TestFreeKeepsSharedBlocks(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + initial := env.bm.FreeCount() + + src := env.newInode() + if _, err := env.mgr.AllocateBlocks(env.ctx, src, 0, 2); err != nil { + t.Fatal(err) + } + if err := env.mgr.MarkAllShared(src); err != nil { + t.Fatal(err) + } + env.write(src) + clone := env.newInode() + if err := env.mgr.CopyExtentsForClone(env.ctx, src, clone); err != nil { + t.Fatal(err) + } + env.write(clone) + blocks, err := env.mgr.DataBlocks(src) + if err != nil { + t.Fatal(err) + } + if err := refs.Share(env.ctx, blocks); err != nil { + t.Fatal(err) + } + + truncate := func(ino *inode.Inode) { + t.Helper() + if err := env.mgr.Truncate(env.ctx, ino, 0); err != nil { + t.Fatal(err) + } + env.write(ino) + } + truncate(src) + if got := env.bm.FreeCount(); got != initial-2 { + t.Fatalf("free blocks after truncating the source = %d, want %d: the clone still maps them", got, initial-2) + } + truncate(clone) + if got := env.bm.FreeCount(); got != initial { + t.Fatalf("free blocks after truncating both = %d, want %d", got, initial) + } + + // Older versions shared blocks with clones without counting them. + legacy := env.newInode() + if _, err := env.mgr.AllocateBlocks(env.ctx, legacy, 0, 1); err != nil { + t.Fatal(err) + } + if err := env.mgr.MarkAllShared(legacy); err != nil { + t.Fatal(err) + } + env.write(legacy) + truncate(legacy) + if got := env.bm.FreeCount(); got != initial-1 { + t.Fatalf("free blocks after truncating an uncounted shared block = %d, want %d", got, initial-1) + } +} diff --git a/internal/extent/tree_edit.go b/internal/extent/tree_edit.go index e4a950ec..3072eefe 100644 --- a/internal/extent/tree_edit.go +++ b/internal/extent/tree_edit.go @@ -714,28 +714,52 @@ func (m *Manager) allocTreeBlocks(ctx context.Context, n int) ([]uint64, error) return out, nil } -// freeBlocks returns blocks to the bitmap. A caller holding an inode lock -// cannot take the bitmap lock, so those frees wait for the next freeBlocks -// call made without it. +// ReleaseBlocks drops one owner from each block and frees the blocks that no +// owner holds any more, such as a block a write just copied away from. +func (m *Manager) ReleaseBlocks(ctx context.Context, blocks []uint64) { + m.freeBlocks(ctx, blocks) +} + +// freeBlocks drops one owner from each block and returns the blocks no owner +// holds to the bitmap. A caller holding an inode lock cannot take the refcount +// or bitmap lock, so those frees wait for the next freeBlocks call made +// without it. func (m *Manager) freeBlocks(ctx context.Context, blocks []uint64) { m.deferredMu.Lock() blocks = append(blocks, m.deferred...) m.deferred = nil m.deferredMu.Unlock() + if m.refs != nil && len(blocks) > 0 { + free, err := m.refs.Release(ctx, blocks) + switch { + case errors.Is(err, lock.ErrDeadlockRisk): + m.deferFree(blocks...) + return + case err != nil: + log.Printf("[extent] release %d blocks: %v; leaving them allocated", len(blocks), err) + return + } + blocks = free + } + for _, b := range blocks { err := m.bitmap.Free(ctx, b) switch { case errors.Is(err, lock.ErrDeadlockRisk): - m.deferredMu.Lock() - m.deferred = append(m.deferred, b) - m.deferredMu.Unlock() + m.deferFree(b) case err != nil: log.Printf("[extent] free block %d: %v", b, err) } } } +func (m *Manager) deferFree(blocks ...uint64) { + m.deferredMu.Lock() + m.deferred = append(m.deferred, blocks...) + m.deferredMu.Unlock() +} + // DiscardPending drops ino's uncommitted extent-tree change after a failed // inode write, freeing the tree blocks allocated for it. Blocks the change // would have released stay allocated because the inode on disk may use them. From 2a50bf0a9ff540a024df3d6ffb6637920fdb2ec0 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:26:46 -0500 Subject: [PATCH 03/18] fix(snapshot): keep each snapshot's blocks in a hidden data inode A snapshot stored no block list, so deleting it used the file's current blocks and a clone of it copied the live file. Each snapshot now owns a hidden inode that shares the blocks it captured; the descriptor names it in bytes 28-31 under the new FlagHasData. Creation shares the blocks before marking the file shared, so later writes copy them. Delete and crash recovery give back the data inode's blocks. Jobs no longer drop when the request context is canceled. fsck accepts data inodes and treats a refcount of 0 as one owner. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- internal/fsck/fsck.go | 2 + internal/fsck/phase3_inodes.go | 5 + internal/fsck/phase5_refcounts.go | 32 +- internal/fsck/phase6_directories.go | 4 +- internal/fsck/phase7_snapshots.go | 52 +- internal/goclusterfs/fs.go | 220 +++++--- internal/goclusterfs/snapshot_test.go | 190 +++++++ internal/inode/inode.go | 1 + internal/snapshot/snapshot.go | 104 ++-- internal/snapshot/snapshot_fuzz_test.go | 22 +- internal/snapshot/snapshot_test.go | 10 +- internal/snapshot/worker.go | 330 +++--------- internal/snapshot/worker_test.go | 635 +++++++++--------------- 14 files changed, 803 insertions(+), 810 deletions(-) create mode 100644 internal/goclusterfs/snapshot_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 960f95f9..8c7ee028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. +- Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. +- Clones made by older versions share blocks without counting them. Newer binaries keep such a block allocated when a file lets go of it, so deleting an old clone or its source leaks its space instead of freeing blocks the other file still uses. Copy the clone with `cp` and delete the old one to avoid the leak. ### Added @@ -26,7 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - UDP multicast cache invalidation between nodes, with optional HMAC authentication (`-broadcast-secret`). - Batched TRIM of freed blocks (`-trim`). - SFB containers for files under 64 KB, disabled: every file uses normal extents and `-sfb` has no effect. A file that an older build stored in an SFB slot stays readable, moves to a full block on its next write or truncate, and frees its slot when deleted. -- Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Copy-on-write does not trigger yet, so a snapshot does not preserve data, and an in-place write to a clone changes the source file. +- Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Read a snapshot by cloning it with `user.snapshot.clone`. - Experimental: `goclusterfs-fsck` consistency checker with a repair mode. It reports errors on healthy volumes: the superblock free counters are not updated after format, and it expects refcount and snapshot sector checksums the runtime does not write. - Benchmark suite (`tests/benchmark`) and an `internal/metrics` package. - Go, Docker, and QEMU VM test suites, with `tests/test-manifest.yaml` and `make test-parity` to track coverage across them. @@ -53,6 +55,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - All nodes wrote the same journal region. A restarting node replayed and then reset the journal of nodes still writing, and no node could replay a crashed node's journal. Each node registry slot now owns a 4 MB journal, and a crashed node's journal is replayed through its slot. - Nodes never sent registry heartbeats, so a node mounting more than 10 seconds after another took the live node's registry slot. - After a partial write into a fallocated range, reads of the other fallocated blocks, and of the rest of the written block, returned old disk contents. They now return zeros. +- Snapshots and clones never copied on write: block refcounts stopped at 1, so a snapshot kept nothing and a write to a clone changed its source. Each snapshot now keeps the blocks it captured in a hidden inode, a write to a shared block copies it first, and a block is freed only when its last owner lets go. +- Deleting a snapshot after its file changed failed with a refcount underflow and left the snapshot deleting forever, because the delete used the file's current blocks. - Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge. - `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects. - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. diff --git a/internal/fsck/fsck.go b/internal/fsck/fsck.go index 74ccd8a9..566cc781 100644 --- a/internal/fsck/fsck.go +++ b/internal/fsck/fsck.go @@ -334,6 +334,7 @@ type Checker struct { dirEntryRefs map[uint32]int // inode -> reference count from dir entries refcountEntries map[uint64]uint32 // block -> refcount from table snapshotInodes map[uint32]bool // inodes referenced by snapshots + snapshotData map[uint32]bool // inodes holding a snapshot's captured extents extentTrees map[uint32]*extentTree } @@ -401,6 +402,7 @@ func New(dev *storage.Device, opts Options) *Checker { dirEntryRefs: make(map[uint32]int), refcountEntries: make(map[uint64]uint32), snapshotInodes: make(map[uint32]bool), + snapshotData: make(map[uint32]bool), extentTrees: make(map[uint32]*extentTree), } } diff --git a/internal/fsck/phase3_inodes.go b/internal/fsck/phase3_inodes.go index f19da2cb..d69cb220 100644 --- a/internal/fsck/phase3_inodes.go +++ b/internal/fsck/phase3_inodes.go @@ -40,6 +40,7 @@ const ( FlagHasExtentTree uint16 = 1 << 0 FlagHasSnapshot uint16 = 1 << 1 FlagDeleted uint16 = 1 << 2 + FlagSnapshotData uint16 = 1 << 6 ) // InodeData holds parsed inode information. @@ -222,6 +223,10 @@ func (c *Checker) checkInodes() error { true) } + if inode.Flags&FlagSnapshotData != 0 { + c.snapshotData[inodeNum] = true + } + // Record extents for cross-validation c.recordInodeExtents(inodeNum, inode) } diff --git a/internal/fsck/phase5_refcounts.go b/internal/fsck/phase5_refcounts.go index 8661b77c..5b6a3fb3 100644 --- a/internal/fsck/phase5_refcounts.go +++ b/internal/fsck/phase5_refcounts.go @@ -40,8 +40,8 @@ const ( // // Checks performed: // - Each sector has valid CRC32C checksum -// - RefCount > 0 for blocks marked allocated in bitmap -// - RefCount == 0 for free blocks +// - RefCount == 0 for free blocks (0 or 1 means one owner, so an +// allocated block may have either) // - RefCount > 1 implies FlagShared should be set in extent // - No refcount overflow (0xFFFFFFFF) // - Cross-validate with extent references from Phase 4 @@ -133,34 +133,16 @@ func (c *Checker) checkRefcounts() error { isAllocatedInBitmap := c.bitmapAllocated[blockNum] refsFromInodes := c.inodeBlocks[blockNum] - if isAllocatedInBitmap { - // Allocated block should have refcount >= 1 - if entry.RefCount == 0 { - c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - "block allocated in bitmap but refcount is 0", - true) - inconsistentEntries++ - } - } else { - // Free block should have refcount == 0 - if entry.RefCount != 0 { - c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - fmt.Sprintf("block free in bitmap but refcount is %d", entry.RefCount), - true) - inconsistentEntries++ - } - } - - // Cross-validate with inode references - if refsFromInodes > 0 && entry.RefCount == 0 { + if !isAllocatedInBitmap && entry.RefCount != 0 { c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), - fmt.Sprintf("referenced by %d inode(s) but refcount is 0", refsFromInodes), + fmt.Sprintf("block free in bitmap but refcount is %d", entry.RefCount), true) + inconsistentEntries++ } - // Check for shared blocks without proper refcount + // An undercounted block is freed while another inode still maps it. if refsFromInodes > 1 && entry.RefCount < uint32(refsFromInodes) { - c.addProblem(PhaseRefcounts, Warning, fmt.Sprintf("block %d", blockNum), + c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("block %d", blockNum), fmt.Sprintf("referenced by %d inodes but refcount is only %d", refsFromInodes, entry.RefCount), true) } diff --git a/internal/fsck/phase6_directories.go b/internal/fsck/phase6_directories.go index fda44c05..76f08bef 100644 --- a/internal/fsck/phase6_directories.go +++ b/internal/fsck/phase6_directories.go @@ -148,8 +148,8 @@ func (c *Checker) checkDirectories() error { // Check for orphaned inodes (in use but not referenced by any directory) for inodeNum := range c.inodeUsed { - if inodeNum == InodeRoot { - continue // Root directory is special + if inodeNum == InodeRoot || c.snapshotData[inodeNum] { + continue } if c.dirEntryRefs[inodeNum] == 0 { orphanedInodes++ diff --git a/internal/fsck/phase7_snapshots.go b/internal/fsck/phase7_snapshots.go index aaf71769..7410bb7e 100644 --- a/internal/fsck/phase7_snapshots.go +++ b/internal/fsck/phase7_snapshots.go @@ -23,6 +23,9 @@ const ( // SnapshotMagic identifies a valid snapshot descriptor. SnapshotMagic uint32 = 0x534E4150 // "SNAP" + // SnapshotFlagHasData marks descriptors whose DataInode holds the captured extents. + SnapshotFlagHasData uint8 = 1 << 4 + // MaxSnapshotNameLen is the maximum length of a snapshot name. MaxSnapshotNameLen = 48 ) @@ -37,20 +40,20 @@ const ( // SnapshotDescriptor holds parsed snapshot information. type SnapshotDescriptor struct { - Magic uint32 - SnapshotID uint64 - ParentID uint64 - TargetInode uint32 - State uint8 - Flags uint8 - ExtentTreeRoot uint64 - TotalBlocks uint64 - TotalSize uint64 - CreateTime int64 - DeleteTime int64 - RefCountJobID uint64 - Name string - Checksum uint32 + Magic uint32 + SnapshotID uint64 + ParentID uint64 + TargetInode uint32 + State uint8 + Flags uint8 + DataInode uint32 + TotalBlocks uint64 + TotalSize uint64 + CreateTime int64 + DeleteTime int64 + RefCountJobID uint64 + Name string + Checksum uint32 } // checkSnapshots validates the snapshot registry (Phase 7). @@ -99,6 +102,7 @@ func (c *Checker) checkSnapshots() error { // Track snapshot IDs and names for duplicate detection seenIDs := make(map[uint64]bool) seenNames := make(map[string]uint64) // name -> snapshot ID + dataInodes := make(map[uint32]bool) for sectorIdx := uint64(0); sectorIdx < totalSectors; sectorIdx++ { sectorOffset := snapshotOffset + sectorIdx*SnapshotSectorSize @@ -197,6 +201,16 @@ func (c *Checker) checkSnapshots() error { c.snapshotInodes[desc.TargetInode] = true } + if desc.Flags&SnapshotFlagHasData != 0 && desc.DataInode != 0 { + if !c.snapshotData[desc.DataInode] { + c.addProblem(PhaseSnapshots, Error, fmt.Sprintf("snapshot %d", desc.SnapshotID), + fmt.Sprintf("data inode %d is not a snapshot data inode", desc.DataInode), + false) + } + dataInodes[desc.DataInode] = true + c.snapshotInodes[desc.DataInode] = true + } + // Validate parent reference (if not root snapshot) if desc.ParentID != 0 && !seenIDs[desc.ParentID] { // Parent might come later - this is a forward reference issue @@ -220,6 +234,14 @@ func (c *Checker) checkSnapshots() error { } } + for num := range c.snapshotData { + if !dataInodes[num] { + c.addProblem(PhaseSnapshots, Warning, fmt.Sprintf("inode %d", num), + "snapshot data inode belongs to no snapshot (left by a crash during snapshot creation)", + true) + } + } + // Compare with superblock counter if activeSnapshots != expectedCount { c.addProblem(PhaseSnapshots, Error, "snapshot count", @@ -247,7 +269,7 @@ func parseSnapshotDescriptor(buf []byte) *SnapshotDescriptor { desc.State = buf[24] desc.Flags = buf[25] // Bytes 26-27 reserved - desc.ExtentTreeRoot = binary.LittleEndian.Uint64(buf[28:36]) + desc.DataInode = binary.LittleEndian.Uint32(buf[28:32]) desc.TotalBlocks = binary.LittleEndian.Uint64(buf[36:44]) desc.TotalSize = binary.LittleEndian.Uint64(buf[44:52]) desc.CreateTime = int64(binary.LittleEndian.Uint64(buf[52:60])) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 9a524dfa..519d3074 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -202,6 +202,9 @@ func NewWithComponents( nodeUUID [16]byte, dataOffset uint64, ) *FS { + if refcountReg != nil { + extentMgr.SetRefcounts(refcountReg) + } return &FS{ dev: dev, lockMgr: lockMgr, @@ -246,7 +249,7 @@ func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uin // Create and start background worker cfg := snapshot.DefaultWorkerConfig() - f.snapshotWkr = snapshot.NewWorker(f.snapshotReg, f.refcountReg, f.extentMgr, f.inodeTbl, cfg) + f.snapshotWkr = snapshot.NewWorker(f.snapshotReg, f.extentMgr, f.inodeTbl, cfg) f.snapshotWkr.Start() // Recover any incomplete jobs from crash @@ -2704,16 +2707,12 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseInodeLock() - for _, cow := range cowDone { - newCount, decErr := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock) - if decErr != nil { - // Log but don't fail - the write succeeded, refcount is secondary - log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", - f.inodeNum, cow.OldPhysBlock, decErr) - } else if newCount == 0 { - log.Printf("[DEBUG] Write inode %d: old block %d refcount reached 0 (can be freed)", - f.inodeNum, cow.OldPhysBlock) + if len(cowDone) > 0 { + copied := make([]uint64, len(cowDone)) + for i, cow := range cowDone { + copied[i] = cow.OldPhysBlock } + f.fs.extentMgr.ReleaseBlocks(ctx, copied) } // Cache the freshly-written inode for subsequent reads @@ -3284,8 +3283,8 @@ func (f *File) Removexattr(ctx context.Context, attr string) syscall.Errno { } // markShared marks every extent shared so writes to either copy use CoW. -func (f *File) markShared(ino *inode.Inode) error { - if err := f.fs.extentMgr.MarkAllShared(ino); err != nil { +func (f *FS) markShared(ino *inode.Inode) error { + if err := f.extentMgr.MarkAllShared(ino); err != nil { return err } ino.Flags |= inode.FlagHasSnapshot @@ -3293,29 +3292,128 @@ func (f *File) markShared(ino *inode.Inode) error { return nil } -// createSnapshot creates a new snapshot of this file. +// shareInto gives dst the extents of inode srcNum, sharing its data blocks. +// Each block gains an owner before dst maps it, and src is marked shared last, +// so from then on writes to either inode copy blocks before changing them. On +// error the caller discards dst, which gives back any blocks it maps. +func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode) error { + src, err := f.inodeTbl.Read(srcNum) + if err != nil { + return err + } + blocks, err := f.extentMgr.DataBlocks(src) + if err != nil { + return err + } + if err := f.refcountReg.Share(ctx, blocks); err != nil { + return err + } + if err := f.extentMgr.CopyExtentsForClone(ctx, src, dst); err != nil { + f.extentMgr.ReleaseBlocks(ctx, blocks) + return err + } + dst.Size = src.Size + if err := f.inodeTbl.Write(ctx, dst); err != nil { + f.extentMgr.DiscardPending(ctx, dst) + // Only a conflict proves dst on disk does not map the blocks. + if errors.Is(err, extent.ErrTreeConflict) { + f.extentMgr.ReleaseBlocks(ctx, blocks) + } + return err + } + _, err = f.updateInode(ctx, srcNum, f.markShared) + return err +} + +// claimInode allocates an inode slot, retrying when another node takes the +// free slot first. +func (f *FS) claimInode(ctx context.Context) (*inode.Inode, error) { + for attempt := 0; ; attempt++ { + num, err := f.inodeTbl.FindFreeSlot() + if err != nil { + return nil, err + } + ino, err := f.inodeTbl.ClaimSlot(ctx, num) + if !errors.Is(err, inode.ErrInodeInUse) || attempt == maxConflictRetries-1 { + return ino, err + } + } +} + +// discardInode gives back the blocks an unlinked inode maps and frees it. The +// empty extent map is committed before any block is released. +func (f *FS) discardInode(ctx context.Context, num uint32) { + _, err := f.updateInode(ctx, num, func(ino *inode.Inode) error { + return f.extentMgr.Truncate(ctx, ino, 0) + }) + if err == nil { + err = f.inodeTbl.Free(ctx, num) + } + if err != nil { + log.Printf("[clone] Failed to discard inode %d: %v", num, err) + } +} + +// dropSnapshot marks a snapshot deleting and queues the job that gives back +// its blocks. A snapshot left deleting is finished by the next mount. +func (f *FS) dropSnapshot(ctx context.Context, snapshotID uint64) error { + if err := f.snapshotReg.Delete(ctx, snapshotID); err != nil { + return err + } + if _, err := f.snapshotWkr.QueueDeleteJob(ctx, snapshotID); err != nil { + log.Printf("[snapshot] Delete of snapshot %d waits for the next mount: %v", snapshotID, err) + } + return nil +} + +// sharableInode reads this file's inode, moving it out of an SFB container +// first: container slots are written in place and cannot be shared. +func (f *File) sharableInode(ctx context.Context) (*inode.Inode, error) { + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || !ino.IsSFBBacked() { + return ino, err + } + if err := f.migrateToExtent(ctx, ino); err != nil { + return nil, err + } + return f.fs.inodeTbl.Read(f.inodeNum) +} + +// createSnapshot creates a new snapshot of this file. The snapshot's data is a +// hidden inode that shares the file's blocks as they are now. func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { if f.fs.snapshotReg == nil || f.fs.snapshotWkr == nil { log.Printf("[DEBUG] createSnapshot: snapshot subsystem not initialized") return syscall.ENOTSUP } + if len(name) >= snapshot.MaxSnapshotNameLen { + return syscall.ENAMETOOLONG + } - // Mark all extents as shared BEFORE creating snapshot (CoW active immediately) - ino, err := f.fs.updateInode(ctx, f.inodeNum, f.markShared) + src, err := f.sharableInode(ctx) if err != nil { - log.Printf("[snapshot] Failed to mark extents shared for inode %d: %v", f.inodeNum, err) + log.Printf("[snapshot] Failed to read inode %d: %v", f.inodeNum, err) return syscall.EIO } - blockNums, err := f.fs.extentMgr.DataBlocks(ino) + data, err := f.fs.claimInode(ctx) if err != nil { - log.Printf("[snapshot] Failed to list blocks for inode %d: %v", f.inodeNum, err) + log.Printf("[snapshot] No inode for the data of a snapshot of inode %d: %v", f.inodeNum, err) + return syscall.ENOSPC + } + data.Mode, data.UID, data.GID = src.Mode, src.UID, src.GID + data.Mtime, data.Ctime = src.Mtime, src.Ctime + data.Flags |= inode.FlagSnapshotData + if err := f.fs.inodeTbl.Write(ctx, data); err != nil { + f.fs.inodeTbl.Free(ctx, data.InodeNum) + log.Printf("[snapshot] Failed to write data inode %d: %v", data.InodeNum, err) return syscall.EIO } desc, err := f.fs.snapshotReg.Create(ctx, name, f.inodeNum, 0, - ino.ExtentTreeRoot, uint64(len(blockNums)), ino.Size, snapshot.FlagReadOnly) + data.InodeNum, src.AllocatedBlocks, src.Size, snapshot.FlagReadOnly) if err != nil { + f.fs.inodeTbl.Free(ctx, data.InodeNum) if errors.Is(err, snapshot.ErrSnapshotExists) { return syscall.EEXIST } @@ -3325,9 +3423,16 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EIO } - // Queue background refcount increment (returns immediately) - if _, err := f.fs.snapshotWkr.QueueCreateJob(ctx, desc.SnapshotID, f.inodeNum, blockNums); err != nil { - log.Printf("[snapshot] Warning: queue job failed: %v", err) + err = f.fs.shareInto(ctx, f.inodeNum, data) + if err == nil { + err = f.fs.snapshotReg.UpdateState(ctx, desc.SnapshotID, snapshot.StateActive) + } + if err != nil { + log.Printf("[snapshot] Failed to create '%s' for inode %d: %v", name, f.inodeNum, err) + if dropErr := f.fs.dropSnapshot(ctx, desc.SnapshotID); dropErr != nil { + log.Printf("[snapshot] Failed to roll back snapshot %d: %v", desc.SnapshotID, dropErr) + } + return syscall.EIO } log.Printf("[snapshot] Created '%s' (ID=%d) for inode %d", name, desc.SnapshotID, f.inodeNum) @@ -3352,21 +3457,11 @@ func (f *File) deleteSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EINVAL } - if err := f.fs.snapshotReg.Delete(ctx, desc.SnapshotID); err != nil { + if err := f.fs.dropSnapshot(ctx, desc.SnapshotID); err != nil { + log.Printf("[snapshot] Failed to delete '%s': %v", name, err) return syscall.EIO } - var blockNums []uint64 - if ino, err := f.fs.inodeTbl.Read(f.inodeNum); err == nil { - if blockNums, err = f.fs.extentMgr.DataBlocks(ino); err != nil { - log.Printf("[snapshot] Failed to list blocks for inode %d: %v", f.inodeNum, err) - } - } - - if _, err := f.fs.snapshotWkr.QueueDeleteJob(ctx, desc.SnapshotID, blockNums); err != nil { - log.Printf("[snapshot] Warning: queue delete job failed: %v", err) - } - log.Printf("[snapshot] Deleting '%s' (ID=%d)", name, desc.SnapshotID) return 0 } @@ -3376,6 +3471,18 @@ func (f *File) deleteSnapshot(ctx context.Context, name string) syscall.Errno { // point Copy-on-Write allocates new blocks for the modified data. func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { log.Printf("[clone] Creating clone of inode %d at %s", f.inodeNum, destPath) + if _, err := f.sharableInode(ctx); err != nil { + log.Printf("[clone] Failed to read inode %d: %v", f.inodeNum, err) + return syscall.EIO + } + return f.cloneInode(ctx, f.inodeNum, destPath) +} + +// cloneInode creates a file at destPath that shares the blocks of inode srcNum. +func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string) syscall.Errno { + if f.fs.refcountReg == nil { + return syscall.ENOTSUP + } // Parse destination path parentPath, cloneName, err := parseClonePath(destPath) @@ -3403,10 +3510,9 @@ func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { return syscall.EEXIST } - // Mark source extents as shared (enables CoW on source) - srcIno, err := f.fs.updateInode(ctx, f.inodeNum, f.markShared) + srcIno, err := f.fs.inodeTbl.Read(srcNum) if err != nil { - log.Printf("[clone] Failed to update source inode: %v", err) + log.Printf("[clone] Failed to read source inode %d: %v", srcNum, err) return syscall.EIO } @@ -3435,47 +3541,24 @@ func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { return syscall.ENOSPC } - // Copy extents with FlagShared set - if err := f.fs.extentMgr.CopyExtentsForClone(ctx, srcIno, cloneIno); err != nil { - f.fs.dirMgr.Remove(ctx, rootIno, cloneName) - f.fs.inodeTbl.Free(ctx, cloneInodeNum) - log.Printf("[clone] Failed to copy extents: %v", err) - return syscall.EIO - } - - // Copy attributes from source now := time.Now().UnixNano() cloneIno.Mode = srcIno.Mode cloneIno.UID = srcIno.UID cloneIno.GID = srcIno.GID - cloneIno.Size = srcIno.Size cloneIno.LinkCount = 1 cloneIno.Flags |= inode.FlagHasSnapshot cloneIno.Atime = now cloneIno.Mtime = now cloneIno.Ctime = now - // Write clone inode - if err := f.fs.inodeTbl.Write(ctx, cloneIno); err != nil { - f.fs.extentMgr.DiscardPending(ctx, cloneIno) + if err := f.fs.shareInto(ctx, srcNum, cloneIno); err != nil { f.fs.dirMgr.Remove(ctx, rootIno, cloneName) - f.fs.inodeTbl.Free(ctx, cloneInodeNum) - log.Printf("[clone] Failed to write clone inode: %v", err) + f.fs.discardInode(ctx, cloneInodeNum) + log.Printf("[clone] Failed to share blocks of inode %d: %v", srcNum, err) return syscall.EIO } - // Queue background refcount increment for shared blocks - blockNums, err := f.fs.extentMgr.DataBlocks(srcIno) - if err != nil { - log.Printf("[clone] Failed to list source blocks: %v", err) - } - if len(blockNums) > 0 && f.fs.snapshotWkr != nil { - if _, err := f.fs.snapshotWkr.QueueCreateJob(ctx, 0, f.inodeNum, blockNums); err != nil { - log.Printf("[clone] Warning: queue refcount job failed: %v", err) - } - } - - log.Printf("[clone] Created '%s' (inode %d) from inode %d", cloneName, cloneInodeNum, f.inodeNum) + log.Printf("[clone] Created '%s' (inode %d) from inode %d", cloneName, cloneInodeNum, srcNum) return 0 } @@ -3507,8 +3590,15 @@ func (f *File) createCloneFromSnapshot(ctx context.Context, data string) syscall log.Printf("[clone] Snapshot '%s' belongs to inode %d, not %d", snapName, desc.TargetInode, f.inodeNum) return syscall.EINVAL } + if desc.State != snapshot.StateActive { + return syscall.EBUSY + } + if !desc.HasData() { + log.Printf("[clone] Snapshot '%s' was made by an older version and holds no data", snapName) + return syscall.ENOTSUP + } - return f.createClone(ctx, destPath) + return f.cloneInode(ctx, desc.DataInode, destPath) } // listSnapshots returns snapshot names for this file. diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go new file mode 100644 index 00000000..04c56661 --- /dev/null +++ b/internal/goclusterfs/snapshot_test.go @@ -0,0 +1,190 @@ +package goclusterfs + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hanwen/go-fuse/v2/fuse" + + "github.com/supporttools/GoClusterFS/internal/bitmap" + "github.com/supporttools/GoClusterFS/internal/dir" + "github.com/supporttools/GoClusterFS/internal/extent" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/refcount" + "github.com/supporttools/GoClusterFS/internal/snapshot" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +const ( + snapFSBitmapOffset = 1 << 20 + snapFSInodeOffset = 2 << 20 + snapFSRefcountOffset = 3 << 20 + snapFSRegistryOffset = 3<<20 + 64<<10 + snapFSRegistrySize = 64 << 10 + snapFSDataOffset = 4 << 20 + snapFSDataBlocks = 60 +) + +func newSnapshotFSImage(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fs.img") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(path, snapFSDataOffset+snapFSDataBlocks*BlockSize); err != nil { + t.Fatal(err) + } + return path +} + +// openSnapshotFS mounts an image with snapshots enabled, as a fresh node would. +func openSnapshotFS(t *testing.T, path string) *FS { + t.Helper() + dev, err := storage.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { dev.Close() }) + + var nodeUUID [16]byte + copy(nodeUUID[:], "testnode12345678") + bm := bitmap.NewRegion(dev, snapFSBitmapOffset, snapFSDataBlocks, nil, nodeUUID) + inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, nil, nodeUUID) + extentMgr := extent.NewManager(dev, bm, nil, snapFSDataOffset) + inodeTbl.SetTreeFlusher(extentMgr) + refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, nil) + dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, nil, snapFSDataOffset) + fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, nodeUUID, snapFSDataOffset) + if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize); err != nil { + t.Fatalf("InitSnapshots: %v", err) + } + t.Cleanup(fsys.snapshotWkr.Stop) + return fsys +} + +func setxattr(t *testing.T, f *File, attr, value string) { + t.Helper() + if errno := f.Setxattr(context.Background(), attr, []byte(value), 0); errno != 0 { + t.Fatalf("setxattr %s=%q on inode %d: %v", attr, value, f.inodeNum, errno) + } +} + +func lookupFile(t *testing.T, fsys *FS, name string) *File { + t.Helper() + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + num, _, err := fsys.dirMgr.Lookup(context.Background(), root, name) + if err != nil { + t.Fatalf("lookup %s: %v", name, err) + } + return &File{fs: fsys, inodeNum: uint32(num)} +} + +func truncateFile(t *testing.T, f *File, size uint64) { + t.Helper() + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_SIZE + in.Size = size + if errno := f.Setattr(context.Background(), nil, in, &fuse.AttrOut{}); errno != 0 { + t.Fatalf("truncate inode %d to %d: %v", f.inodeNum, size, errno) + } +} + +// checkFile compares the whole file, including its size, with want. +func checkFile(t *testing.T, f *File, want []byte) { + t.Helper() + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.Size != uint64(len(want)) { + t.Fatalf("inode %d: size %d, want %d", f.inodeNum, ino.Size, len(want)) + } + checkAt(t, f, want, 0) +} + +func overwritten(base []byte, off int, data []byte) []byte { + out := bytes.Clone(base) + copy(out[off:], data) + return out +} + +func TestSnapshotKeepsDataThroughWrites(t *testing.T) { + path := newSnapshotFSImage(t) + fsys := openSnapshotFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatalf("InitRoot: %v", err) + } + original := pattern(1, 3*BlockSize) + + t.Run("SnapshotSurvivesWrites", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "before") + + writeAt(t, f, pattern(2, 4096), BlockSize+100) + writeAt(t, f, pattern(3, 2*BlockSize), 3*BlockSize) + truncateFile(t, f, BlockSize+BlockSize/2) + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + + live := overwritten(original[:BlockSize+BlockSize/2], BlockSize+100, pattern(2, 4096)) + for _, fsys := range []*FS{fsys, openSnapshotFS(t, path)} { + checkFile(t, lookupFile(t, fsys, "before-clone"), original) + checkFile(t, &File{fs: fsys, inodeNum: f.inodeNum}, live) + } + }) + + t.Run("CloneWritesStayInClone", func(t *testing.T) { + src := newRegularFile(t, fsys) + writeAt(t, src, original[:2*BlockSize], 0) + setxattr(t, src, "user.clone", "/clone") + clone := lookupFile(t, fsys, "clone") + + writeAt(t, clone, pattern(4, 100), 10) + writeAt(t, src, pattern(5, 100), BlockSize+10) + + for _, fsys := range []*FS{fsys, openSnapshotFS(t, path)} { + checkFile(t, &File{fs: fsys, inodeNum: src.inodeNum}, overwritten(original[:2*BlockSize], BlockSize+10, pattern(5, 100))) + checkFile(t, &File{fs: fsys, inodeNum: clone.inodeNum}, overwritten(original[:2*BlockSize], 10, pattern(4, 100))) + } + }) + + t.Run("DeleteAfterOverwriteFreesSnapshotBlocks", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original[:2*BlockSize], 0) + free := fsys.bitmapMgr.FreeCount() + setxattr(t, f, "user.snapshot", "temp") + + writeAt(t, f, pattern(6, 2*BlockSize), 0) + if got := fsys.bitmapMgr.FreeCount(); got != free-2 { + t.Fatalf("free blocks after overwriting 2 shared blocks = %d, want %d", got, free-2) + } + + if errno := f.Removexattr(ctx, "user.snapshot.temp"); errno != 0 { + t.Fatalf("delete snapshot: %v", errno) + } + deadline := time.Now().Add(10 * time.Second) + for { + _, err := fsys.snapshotReg.GetByName(ctx, "temp") + if errors.Is(err, snapshot.ErrSnapshotNotFound) { + break + } + if time.Now().After(deadline) { + t.Fatalf("snapshot still listed 10s after delete: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, f, pattern(6, 2*BlockSize)) + }) +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 8daf0578..230d3316 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -64,6 +64,7 @@ const ( FlagSync uint16 = 1 << 3 // Synchronous updates FlagImmutable uint16 = 1 << 4 // Cannot be modified FlagAppend uint16 = 1 << 5 // Append-only + FlagSnapshotData uint16 = 1 << 6 // Holds a snapshot's frozen extents; in no directory ) // Errors diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index ddc0f9a8..af72ac71 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -105,6 +105,11 @@ const ( // FlagAutoDelete indicates the snapshot should be auto-deleted // when no longer referenced. FlagAutoDelete Flags = 1 << 3 + + // FlagHasData marks the current descriptor format: DataInode names a hidden + // inode that maps the blocks the snapshot captured. Descriptors written by + // older versions lack it and hold no data. + FlagHasData Flags = 1 << 4 ) // Errors @@ -135,7 +140,8 @@ var ( // 24 1 State (Creating/Active/Deleting) // 25 1 Flags (ReadOnly/Writable/Hidden) // 26 2 Reserved1 -// 28 8 ExtentTreeRoot (copied from inode at snapshot time) +// 28 4 DataInode (hidden inode with the captured extents; only with FlagHasData) +// 32 4 Reserved2 (older versions wrote an unused extent tree root at 28-35) // 36 8 TotalBlocks (block count at snapshot time) // 44 8 TotalSize (file size at snapshot time) // 52 8 CreateTime (Unix timestamp) @@ -144,21 +150,22 @@ var ( // 76 48 Name (null-terminated UTF-8 string) // 124 4 Checksum (CRC32C of bytes 0-123) type Descriptor struct { - Magic uint32 // 0x534E4150 "SNAP" - SnapshotID uint64 // Unique snapshot identifier - ParentID uint64 // Parent snapshot (0 for root) - TargetInode uint32 // Inode number of snapshotted file - State State // Current state - Flags Flags // Snapshot flags - Reserved1 uint16 // Alignment padding - ExtentTreeRoot uint64 // Extent tree root block - TotalBlocks uint64 // Number of blocks at snapshot time - TotalSize uint64 // File size at snapshot time - CreateTime int64 // Creation timestamp - DeleteTime int64 // Deletion timestamp (0 if active) - RefCountJobID uint64 // Background refcount job ID - Name [MaxSnapshotNameLen]byte // Snapshot name - Checksum uint32 // CRC32C checksum + Magic uint32 // 0x534E4150 "SNAP" + SnapshotID uint64 // Unique snapshot identifier + ParentID uint64 // Parent snapshot (0 for root) + TargetInode uint32 // Inode number of snapshotted file + State State // Current state + Flags Flags // Snapshot flags + Reserved1 uint16 // Alignment padding + DataInode uint32 // Hidden inode holding the captured extents + Reserved2 uint32 // Reserved + TotalBlocks uint64 // Number of blocks at snapshot time + TotalSize uint64 // File size at snapshot time + CreateTime int64 // Creation timestamp + DeleteTime int64 // Deletion timestamp (0 if active) + RefCountJobID uint64 // Background refcount job ID + Name [MaxSnapshotNameLen]byte // Snapshot name + Checksum uint32 // CRC32C checksum } // IsValid returns true if this is a valid snapshot descriptor. @@ -166,6 +173,11 @@ func (d *Descriptor) IsValid() bool { return d.Magic == DescriptorMagic && d.State != StateInvalid } +// HasData reports whether the snapshot captured data that can be cloned. +func (d *Descriptor) HasData() bool { + return d.Flags&FlagHasData != 0 && d.DataInode != 0 +} + // IsFree returns true if this slot is free (can be reused). func (d *Descriptor) IsFree() bool { return d.Magic == 0 || d.State == StateInvalid @@ -247,14 +259,14 @@ func NewRegistry(dev *storage.Device, offset, size uint64, lockMgr *lock.Manager // - name: Human-readable snapshot name // - targetInode: Inode number being snapshotted // - parentID: Parent snapshot ID (0 for root snapshot) -// - extentTreeRoot: Extent tree root block from the inode +// - dataInode: Hidden inode holding the captured extents (0 for none) // - totalBlocks: Number of blocks at snapshot time // - totalSize: File size at snapshot time // - flags: Snapshot flags (ReadOnly, Writable, etc.) // // Returns the new snapshot ID and descriptor. func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, - parentID uint64, extentTreeRoot uint64, totalBlocks uint64, + parentID uint64, dataInode uint32, totalBlocks uint64, totalSize uint64, flags Flags) (*Descriptor, error) { if len(name) >= MaxSnapshotNameLen { @@ -283,18 +295,20 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, snapshotID := r.nextID r.nextID++ - // Create the descriptor + if dataInode != 0 { + flags |= FlagHasData + } desc := &Descriptor{ - Magic: DescriptorMagic, - SnapshotID: snapshotID, - ParentID: parentID, - TargetInode: targetInode, - State: StateCreating, // Will be marked Active after refcount job - Flags: flags | FlagReadOnly, - ExtentTreeRoot: extentTreeRoot, - TotalBlocks: totalBlocks, - TotalSize: totalSize, - CreateTime: time.Now().Unix(), + Magic: DescriptorMagic, + SnapshotID: snapshotID, + ParentID: parentID, + TargetInode: targetInode, + State: StateCreating, // Will be marked Active after refcount job + Flags: flags | FlagReadOnly, + DataInode: dataInode, + TotalBlocks: totalBlocks, + TotalSize: totalSize, + CreateTime: time.Now().Unix(), } if err := desc.SetName(name); err != nil { return nil, err @@ -787,20 +801,21 @@ func decodeDescriptor(sectorData []byte, descIdx uint64) (*Descriptor, error) { } desc := &Descriptor{ - Magic: magic, - SnapshotID: binary.LittleEndian.Uint64(data[4:12]), - ParentID: binary.LittleEndian.Uint64(data[12:20]), - TargetInode: binary.LittleEndian.Uint32(data[20:24]), - State: State(data[24]), - Flags: Flags(data[25]), - Reserved1: binary.LittleEndian.Uint16(data[26:28]), - ExtentTreeRoot: binary.LittleEndian.Uint64(data[28:36]), - TotalBlocks: binary.LittleEndian.Uint64(data[36:44]), - TotalSize: binary.LittleEndian.Uint64(data[44:52]), - CreateTime: int64(binary.LittleEndian.Uint64(data[52:60])), - DeleteTime: int64(binary.LittleEndian.Uint64(data[60:68])), - RefCountJobID: binary.LittleEndian.Uint64(data[68:76]), - Checksum: stored, + Magic: magic, + SnapshotID: binary.LittleEndian.Uint64(data[4:12]), + ParentID: binary.LittleEndian.Uint64(data[12:20]), + TargetInode: binary.LittleEndian.Uint32(data[20:24]), + State: State(data[24]), + Flags: Flags(data[25]), + Reserved1: binary.LittleEndian.Uint16(data[26:28]), + DataInode: binary.LittleEndian.Uint32(data[28:32]), + Reserved2: binary.LittleEndian.Uint32(data[32:36]), + TotalBlocks: binary.LittleEndian.Uint64(data[36:44]), + TotalSize: binary.LittleEndian.Uint64(data[44:52]), + CreateTime: int64(binary.LittleEndian.Uint64(data[52:60])), + DeleteTime: int64(binary.LittleEndian.Uint64(data[60:68])), + RefCountJobID: binary.LittleEndian.Uint64(data[68:76]), + Checksum: stored, } copy(desc.Name[:], data[76:124]) @@ -829,7 +844,8 @@ func encodeDescriptor(sectorData []byte, descIdx uint64, desc *Descriptor) { data[24] = byte(desc.State) data[25] = byte(desc.Flags) binary.LittleEndian.PutUint16(data[26:28], desc.Reserved1) - binary.LittleEndian.PutUint64(data[28:36], desc.ExtentTreeRoot) + binary.LittleEndian.PutUint32(data[28:32], desc.DataInode) + binary.LittleEndian.PutUint32(data[32:36], desc.Reserved2) binary.LittleEndian.PutUint64(data[36:44], desc.TotalBlocks) binary.LittleEndian.PutUint64(data[44:52], desc.TotalSize) binary.LittleEndian.PutUint64(data[52:60], uint64(desc.CreateTime)) diff --git a/internal/snapshot/snapshot_fuzz_test.go b/internal/snapshot/snapshot_fuzz_test.go index f16c4c81..3b167ffe 100644 --- a/internal/snapshot/snapshot_fuzz_test.go +++ b/internal/snapshot/snapshot_fuzz_test.go @@ -10,17 +10,17 @@ import ( func FuzzDecodeDescriptor(f *testing.F) { active := Descriptor{ - Magic: DescriptorMagic, - SnapshotID: 7, - ParentID: 3, - TargetInode: 42, - State: StateActive, - Flags: FlagReadOnly, - ExtentTreeRoot: 1234, - TotalBlocks: 5, - TotalSize: 5 << 20, - CreateTime: 1_700_000_000, - RefCountJobID: 9, + Magic: DescriptorMagic, + SnapshotID: 7, + ParentID: 3, + TargetInode: 42, + State: StateActive, + Flags: FlagReadOnly | FlagHasData, + DataInode: 1234, + TotalBlocks: 5, + TotalSize: 5 << 20, + CreateTime: 1_700_000_000, + RefCountJobID: 9, } if err := active.SetName("nightly"); err != nil { f.Fatal(err) diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index f38e9ec7..7d3180f1 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -257,8 +257,8 @@ func TestRegistryCreate(t *testing.T) { if desc.GetName() != "snap1" { t.Errorf("Name = %q, want %q", desc.GetName(), "snap1") } - if desc.ExtentTreeRoot != 1000 { - t.Errorf("ExtentTreeRoot = %d, want 1000", desc.ExtentTreeRoot) + if desc.DataInode != 1000 { + t.Errorf("DataInode = %d, want 1000", desc.DataInode) } if desc.TotalBlocks != 50 { t.Errorf("TotalBlocks = %d, want 50", desc.TotalBlocks) @@ -681,8 +681,8 @@ func TestRegistryPersistence(t *testing.T) { if desc.State != StateActive { t.Errorf("State = %v, want %v", desc.State, StateActive) } - if desc.ExtentTreeRoot != 1000 { - t.Errorf("ExtentTreeRoot = %d, want 1000", desc.ExtentTreeRoot) + if desc.DataInode != 1000 { + t.Errorf("DataInode = %d, want 1000", desc.DataInode) } } @@ -740,7 +740,7 @@ func TestMultipleSectors(t *testing.T) { for i := 0; i < 50; i++ { name := fmt.Sprintf("snap%d", i) _, err := registry.Create(ctx, name, uint32(100+i), InvalidSnapshotID, - uint64(1000+i), uint64(50+i), uint64(1024*(i+1)), FlagReadOnly) + uint32(1000+i), uint64(50+i), uint64(1024*(i+1)), FlagReadOnly) if err != nil { t.Fatalf("Create(%s) failed: %v", name, err) } diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index 63b0652e..926ef824 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -1,44 +1,30 @@ // Package snapshot implements snapshot management for GoClusterFS. // -// worker.go implements the background worker for asynchronous refcount updates. -// When a snapshot is created, the descriptor is written immediately (O(1)), -// and the refcount increments are queued for background processing. +// worker.go implements the background worker that deletes snapshots and +// cleans up snapshots a crash left half created or half deleted. package snapshot import ( "context" + "errors" "fmt" "log" "sync" "sync/atomic" "time" - "github.com/supporttools/GoClusterFS/internal/bitmap" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" - "github.com/supporttools/GoClusterFS/internal/refcount" ) -// JobType represents the type of background job. -type JobType uint8 +// ErrWorkerStopped is returned when a job is queued on a stopped worker. +var ErrWorkerStopped = errors.New("snapshot worker stopped") -const ( - // JobTypeCreate processes refcount increments for snapshot creation. - JobTypeCreate JobType = 1 - - // JobTypeDelete processes refcount decrements for snapshot deletion. - JobTypeDelete JobType = 2 -) - -// Job represents a background refcount job. +// Job is a queued snapshot deletion. type Job struct { ID uint64 // Unique job identifier - Type JobType // Create or Delete - SnapshotID uint64 // Snapshot being processed - InodeNum uint32 // Inode number of the source file - BlockNums []uint64 // Block numbers to process + SnapshotID uint64 // Snapshot being deleted StartTime time.Time // When job was queued - Retries int // Number of retry attempts } // JobResult contains the result of a completed job. @@ -50,14 +36,12 @@ type JobResult struct { Duration time.Duration } -// Worker processes background refcount jobs for snapshots. +// Worker deletes snapshots in the background. type Worker struct { mu sync.RWMutex registry *Registry - refRegion *refcount.Region extentMgr *extent.Manager inodeTable *inode.Table - bitmapMgr *bitmap.Region // For freeing blocks when refcount reaches 0 jobQueue chan *Job resultChan chan *JobResult @@ -91,17 +75,9 @@ func DefaultWorkerConfig() WorkerConfig { } } -// NewWorker creates a new background worker. -// Note: bitmapMgr can be nil if block freeing is handled elsewhere, but for -// proper snapshot deletion with block reclamation, it should be provided. -func NewWorker(registry *Registry, refRegion *refcount.Region, extentMgr *extent.Manager, inodeTable *inode.Table, cfg WorkerConfig) *Worker { - return NewWorkerWithBitmap(registry, refRegion, extentMgr, inodeTable, nil, cfg) -} - -// NewWorkerWithBitmap creates a new background worker with bitmap for block freeing. -// When a block's refcount reaches 0 during snapshot deletion, the bitmap is used -// to mark the block as free for reallocation. -func NewWorkerWithBitmap(registry *Registry, refRegion *refcount.Region, extentMgr *extent.Manager, inodeTable *inode.Table, bitmapMgr *bitmap.Region, cfg WorkerConfig) *Worker { +// NewWorker creates a new background worker. Blocks released by a deleted +// snapshot are freed through extentMgr, which must have refcounts set. +func NewWorker(registry *Registry, extentMgr *extent.Manager, inodeTable *inode.Table, cfg WorkerConfig) *Worker { if cfg.QueueSize <= 0 { cfg.QueueSize = 100 } @@ -111,10 +87,8 @@ func NewWorkerWithBitmap(registry *Registry, refRegion *refcount.Region, extentM w := &Worker{ registry: registry, - refRegion: refRegion, extentMgr: extentMgr, inodeTable: inodeTable, - bitmapMgr: bitmapMgr, jobQueue: make(chan *Job, cfg.QueueSize), resultChan: make(chan *JobResult, cfg.QueueSize), stopChan: make(chan struct{}), @@ -155,72 +129,32 @@ func (w *Worker) Stop() { log.Printf("[snapshot/worker] Stopped") } -// QueueCreateJob queues a refcount increment job for snapshot creation. -// Returns the job ID immediately. -func (w *Worker) QueueCreateJob(ctx context.Context, snapshotID uint64, inodeNum uint32, blockNums []uint64) (uint64, error) { - // The snapshot descriptor already exists, so the job must be queued even if the caller gives up. - ctx = context.WithoutCancel(ctx) - +// QueueDeleteJob queues the deletion of a snapshot that is already marked +// deleting. The job outlives the request that queued it, so ctx only carries +// values; a full queue makes the caller wait. +func (w *Worker) QueueDeleteJob(ctx context.Context, snapshotID uint64) (uint64, error) { if !w.running.Load() { - return 0, fmt.Errorf("worker not running") - } - - jobID := w.nextJobID.Add(1) - 1 - job := &Job{ - ID: jobID, - Type: JobTypeCreate, - SnapshotID: snapshotID, - InodeNum: inodeNum, - BlockNums: blockNums, - StartTime: time.Now(), + return 0, ErrWorkerStopped } - - // Store job ID in snapshot descriptor - if err := w.registry.SetRefCountJobID(ctx, snapshotID, jobID); err != nil { - return 0, fmt.Errorf("failed to set job ID: %w", err) - } - - select { - case w.jobQueue <- job: - log.Printf("[snapshot/worker] Queued create job %d for snapshot %d (%d blocks)", - jobID, snapshotID, len(blockNums)) - return jobID, nil - default: - return 0, fmt.Errorf("job queue full") - } -} - -// QueueDeleteJob queues a refcount decrement job for snapshot deletion. -// Returns the job ID immediately. -func (w *Worker) QueueDeleteJob(ctx context.Context, snapshotID uint64, blockNums []uint64) (uint64, error) { - // The snapshot descriptor already exists, so the job must be queued even if the caller gives up. ctx = context.WithoutCancel(ctx) - if !w.running.Load() { - return 0, fmt.Errorf("worker not running") - } - jobID := w.nextJobID.Add(1) - 1 job := &Job{ ID: jobID, - Type: JobTypeDelete, SnapshotID: snapshotID, - BlockNums: blockNums, StartTime: time.Now(), } - // Store job ID in snapshot descriptor if err := w.registry.SetRefCountJobID(ctx, snapshotID, jobID); err != nil { return 0, fmt.Errorf("failed to set job ID: %w", err) } select { case w.jobQueue <- job: - log.Printf("[snapshot/worker] Queued delete job %d for snapshot %d (%d blocks)", - jobID, snapshotID, len(blockNums)) + log.Printf("[snapshot/worker] Queued delete job %d for snapshot %d", jobID, snapshotID) return jobID, nil - default: - return 0, fmt.Errorf("job queue full") + case <-w.stopChan: + return 0, ErrWorkerStopped } } @@ -242,21 +176,10 @@ func (w *Worker) workerLoop(workerID int) { } } -// processJob executes a single refcount job. +// processJob executes a single job. func (w *Worker) processJob(job *Job) *JobResult { - ctx := context.Background() start := time.Now() - - var err error - switch job.Type { - case JobTypeCreate: - err = w.processCreateJob(ctx, job) - case JobTypeDelete: - err = w.processDeleteJob(ctx, job) - default: - err = fmt.Errorf("unknown job type: %d", job.Type) - } - + err := w.processDeleteJob(context.Background(), job) return &JobResult{ JobID: job.ID, SnapshotID: job.SnapshotID, @@ -266,128 +189,71 @@ func (w *Worker) processJob(job *Job) *JobResult { } } -// processCreateJob increments refcounts and transitions snapshot to Active. -func (w *Worker) processCreateJob(ctx context.Context, job *Job) error { - if len(job.BlockNums) == 0 { - // No blocks to process, just transition state - return w.registry.UpdateState(ctx, job.SnapshotID, StateActive) - } - - // Increment refcounts in batches for efficiency - const batchSize = 512 // Process 512 blocks per batch - for i := 0; i < len(job.BlockNums); i += batchSize { - end := i + batchSize - if end > len(job.BlockNums) { - end = len(job.BlockNums) - } - - batch := job.BlockNums[i:end] - if _, err := w.refRegion.IncrementBatch(ctx, batch); err != nil { - return fmt.Errorf("failed to increment refcounts (batch %d-%d): %w", i, end, err) - } - } - - // Transition snapshot to Active state - if err := w.registry.UpdateState(ctx, job.SnapshotID, StateActive); err != nil { - return fmt.Errorf("failed to update snapshot state: %w", err) - } - - log.Printf("[snapshot/worker] Snapshot %d now active (%d blocks processed)", - job.SnapshotID, len(job.BlockNums)) - return nil -} - -// processDeleteJob handles snapshot deletion with merge-up for middle snapshots. -// -// For leaf snapshots (no children): -// - Decrement all block refcounts -// - Free blocks where refcount reaches 0 -// -// For middle snapshots (has children): -// - Re-parent children to this snapshot's parent (merge-up) -// - Decrement refcounts (shared blocks won't reach 0) -// - Free blocks that reach refcount 0 +// processDeleteJob re-parents the snapshot's children to its parent, gives +// back the blocks it captured, and removes its descriptor. func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { - // First, get the snapshot to find its parent snap, err := w.registry.Get(ctx, job.SnapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + return nil + } if err != nil { return fmt.Errorf("failed to get snapshot %d: %w", job.SnapshotID, err) } - // Check for children (determines if this is a leaf or middle snapshot) children, err := w.registry.GetChildren(ctx, job.SnapshotID) if err != nil { return fmt.Errorf("failed to get children for snapshot %d: %w", job.SnapshotID, err) } - - // Handle merge-up: re-parent children to grandparent - if len(children) > 0 { - log.Printf("[snapshot/worker] Snapshot %d has %d children, performing merge-up to parent %d", - job.SnapshotID, len(children), snap.ParentID) - - for _, child := range children { - if err := w.registry.UpdateParentID(ctx, child.SnapshotID, snap.ParentID); err != nil { - return fmt.Errorf("failed to re-parent snapshot %d: %w", child.SnapshotID, err) - } - log.Printf("[snapshot/worker] Re-parented snapshot %d from %d to %d", - child.SnapshotID, job.SnapshotID, snap.ParentID) + for _, child := range children { + if err := w.registry.UpdateParentID(ctx, child.SnapshotID, snap.ParentID); err != nil { + return fmt.Errorf("failed to re-parent snapshot %d: %w", child.SnapshotID, err) } + log.Printf("[snapshot/worker] Re-parented snapshot %d from %d to %d", + child.SnapshotID, job.SnapshotID, snap.ParentID) } - // Decrement refcounts and free blocks that reach 0 - var freedCount int - if len(job.BlockNums) > 0 { - freedCount, err = w.decrementAndFreeBlocks(ctx, job.BlockNums) - if err != nil { - return fmt.Errorf("failed to decrement/free blocks: %w", err) + if snap.HasData() { + if err := w.ReleaseData(ctx, snap.DataInode); err != nil { + return fmt.Errorf("failed to release data of snapshot %d: %w", job.SnapshotID, err) } } - // Remove the snapshot descriptor if err := w.registry.Remove(ctx, job.SnapshotID); err != nil { return fmt.Errorf("failed to remove snapshot: %w", err) } - log.Printf("[snapshot/worker] Snapshot %d deleted (%d blocks processed, %d freed)", - job.SnapshotID, len(job.BlockNums), freedCount) + log.Printf("[snapshot/worker] Snapshot %d deleted", job.SnapshotID) return nil } -// decrementAndFreeBlocks decrements refcounts for blocks and frees those that reach 0. -// Returns the count of blocks freed. -func (w *Worker) decrementAndFreeBlocks(ctx context.Context, blockNums []uint64) (int, error) { - const batchSize = 512 - freedCount := 0 - - for i := 0; i < len(blockNums); i += batchSize { - end := i + batchSize - if end > len(blockNums) { - end = len(blockNums) - } - - batch := blockNums[i:end] - newCounts, err := w.refRegion.DecrementBatch(ctx, batch) +// ReleaseData gives back the blocks held by a snapshot's data inode and frees +// the inode. The empty extent map is committed before any block is released, +// so a crash leaks blocks at worst and calling it again is safe. +func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { + const attempts = 5 + for attempt := 1; ; attempt++ { + ino, err := w.inodeTable.Read(num) if err != nil { - return freedCount, fmt.Errorf("failed to decrement refcounts (batch %d-%d): %w", i, end, err) + return err } - - // Free blocks that reached refcount 0 - if w.bitmapMgr != nil { - for j, newCount := range newCounts { - if newCount == 0 { - blockNum := batch[j] - if err := w.bitmapMgr.Free(ctx, blockNum); err != nil { - // Log but don't fail - block leak is better than corruption - log.Printf("[snapshot/worker] Warning: failed to free block %d: %v", blockNum, err) - } else { - freedCount++ - } - } - } + if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { + // Already freed, and possibly reused by a regular file since. + return nil + } + if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil { + w.extentMgr.DiscardPending(ctx, ino) + return err + } + err = w.inodeTable.Write(ctx, ino) + if err == nil { + break + } + w.extentMgr.DiscardPending(ctx, ino) + if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts { + return err } } - - return freedCount, nil + return w.inodeTable.Free(ctx, num) } // resultHandler processes completed job results. @@ -416,83 +282,43 @@ func (w *Worker) resultHandler() { } } -// RecoverIncompleteJobs scans for snapshots in StateCreating or StateDeleting -// and re-queues their background jobs. Call this on mount/startup. +// RecoverIncompleteJobs deletes snapshots that a crash left creating or +// deleting. A snapshot still creating never became visible as active, so it +// is rolled back. Call this on mount, after Start. func (w *Worker) RecoverIncompleteJobs(ctx context.Context) error { w.mu.Lock() defer w.mu.Unlock() - // Scan registry for incomplete snapshots - stats, err := w.registry.Scan(ctx) - if err != nil { - return fmt.Errorf("failed to scan registry: %w", err) - } - - if stats.CreatingCount == 0 && stats.DeletingCount == 0 { - log.Printf("[snapshot/worker] No incomplete jobs to recover") - return nil - } - - log.Printf("[snapshot/worker] Recovering %d creating + %d deleting snapshots", - stats.CreatingCount, stats.DeletingCount) - - // Get all snapshots and process incomplete ones snapshots, err := w.registry.List(ctx) if err != nil { return fmt.Errorf("failed to list snapshots: %w", err) } + var recovered int for _, snap := range snapshots { switch snap.State { case StateCreating: - if err := w.recoverCreateJob(ctx, snap); err != nil { - log.Printf("[snapshot/worker] Failed to recover create job for snapshot %d: %v", - snap.SnapshotID, err) + if err := w.registry.Delete(ctx, snap.SnapshotID); err != nil { + log.Printf("[snapshot/worker] Failed to roll back snapshot %d: %v", snap.SnapshotID, err) + continue } - case StateDeleting: - if err := w.recoverDeleteJob(ctx, snap); err != nil { - log.Printf("[snapshot/worker] Failed to recover delete job for snapshot %d: %v", - snap.SnapshotID, err) - } + default: + continue } + if _, err := w.QueueDeleteJob(ctx, snap.SnapshotID); err != nil { + log.Printf("[snapshot/worker] Failed to queue cleanup of snapshot %d: %v", snap.SnapshotID, err) + continue + } + recovered++ } - return nil -} - -// recoverCreateJob recovers a snapshot stuck in StateCreating. -func (w *Worker) recoverCreateJob(ctx context.Context, snap *Descriptor) error { - // Read the inode to get current block list - ino, err := w.inodeTable.Read(snap.TargetInode) - if err != nil { - return fmt.Errorf("failed to read inode %d: %w", snap.TargetInode, err) - } - - // Collect all block numbers from the inode - blockNums, err := w.extentMgr.DataBlocks(ino) - if err != nil { - return fmt.Errorf("failed to list blocks of inode %d: %w", snap.TargetInode, err) + if recovered == 0 { + log.Printf("[snapshot/worker] No incomplete jobs to recover") + } else { + log.Printf("[snapshot/worker] Cleaning up %d incomplete snapshot(s)", recovered) } - - // Re-queue the create job - // Note: This may result in some blocks being incremented twice if crash - // happened mid-way. This is safe - it means slightly higher refcounts - // that will be cleaned up when the snapshot is deleted. - _, err = w.QueueCreateJob(ctx, snap.SnapshotID, snap.TargetInode, blockNums) - return err -} - -// recoverDeleteJob recovers a snapshot stuck in StateDeleting. -func (w *Worker) recoverDeleteJob(ctx context.Context, snap *Descriptor) error { - // For deletion, we need to read the snapshot's extent tree (not the current inode) - // Since we stored ExtentTreeRoot at snapshot time, we use that. - // However, the blocks may have already been decremented, so we need to be careful. - - // For now, just remove the descriptor - the refcounts were either - // already decremented or will be off by 1 (leak) which is acceptable - // for crash recovery. - return w.registry.Remove(ctx, snap.SnapshotID) + return nil } // PendingJobs returns the number of jobs waiting in the queue. diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index 0426afb1..90f3e213 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -2,12 +2,14 @@ package snapshot import ( "context" + "errors" "fmt" "os" "path/filepath" "testing" "time" + "github.com/supporttools/GoClusterFS/internal/bitmap" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" "github.com/supporttools/GoClusterFS/internal/refcount" @@ -18,11 +20,12 @@ const ( testBlockSize = 1024 * 1024 // 1MB testDeviceSize = 100 * testBlockSize testDataBlocks = 50 + testBitmapStart = uint64(60 * testBlockSize) + testInodeStart = uint64(70 * testBlockSize) + testInodeSize = uint64(10 * testBlockSize) testRefcountStart = uint64(80 * testBlockSize) testSnapshotStart = uint64(90 * testBlockSize) testSnapshotSize = uint64(5 * 4096) // 5 sectors - testInodeStart = uint64(70 * testBlockSize) - testInodeSize = uint64(10 * testBlockSize) ) // setupTestDevice creates a temporary device for testing. @@ -61,73 +64,169 @@ func setupTestDevice(t *testing.T) (*storage.Device, string, func()) { return dev, devPath, cleanup } -// setupTestComponents creates all components needed for worker testing. -const jobTimeout = 10 * time.Second +// workerEnv holds a small filesystem: data blocks start at device offset 0. +type workerEnv struct { + t *testing.T + ctx context.Context + reg *Registry + refs *refcount.Region + bm *bitmap.Region + tbl *inode.Table + ext *extent.Manager + w *Worker + done chan *JobResult +} -// waitForJob polls done until the worker's background job has finished. -func waitForJob(t *testing.T, done func() bool) { +func newWorkerEnv(t *testing.T) *workerEnv { t.Helper() - deadline := time.Now().Add(jobTimeout) - for !done() { - if time.Now().After(deadline) { - t.Fatalf("Background job did not finish within %v", jobTimeout) - } - time.Sleep(10 * time.Millisecond) - } + dev, _, cleanup := setupTestDevice(t) + t.Cleanup(cleanup) + + var nodeUUID [16]byte + env := &workerEnv{ + t: t, + ctx: context.Background(), + reg: NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil), + refs: refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil), + bm: bitmap.NewRegion(dev, testBitmapStart, testDataBlocks, nil, nodeUUID), + tbl: inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID), + done: make(chan *JobResult, 64), + } + if err := env.reg.Initialize(env.ctx); err != nil { + t.Fatalf("initialize registry: %v", err) + } + env.ext = extent.NewManager(dev, env.bm, nil, 0) + env.ext.SetRefcounts(env.refs) + env.tbl.SetTreeFlusher(env.ext) + + cfg := DefaultWorkerConfig() + cfg.OnJobComplete = func(r *JobResult) { env.done <- r } + env.w = NewWorker(env.reg, env.ext, env.tbl, cfg) + return env } -func snapshotActive(reg *Registry, id uint64) func() bool { - return func() bool { - desc, err := reg.Get(context.Background(), id) - return err == nil && desc.State == StateActive - } +func (env *workerEnv) start() { + env.w.Start() + env.t.Cleanup(env.w.Stop) } -func snapshotRemoved(reg *Registry, id uint64) func() bool { - return func() bool { - _, err := reg.Get(context.Background(), id) - return err == ErrSnapshotNotFound +func (env *workerEnv) claim() *inode.Inode { + env.t.Helper() + num, err := env.tbl.FindFreeSlot() + if err != nil { + env.t.Fatal(err) + } + ino, err := env.tbl.ClaimSlot(env.ctx, num) + if err != nil { + env.t.Fatal(err) } + ino.Mode = inode.S_IFREG | 0o644 + return ino } -func setupTestComponents(t *testing.T, dev *storage.Device) (*Registry, *refcount.Region, *extent.Manager, *inode.Table) { - t.Helper() - - // Create refcount region - refReg := refcount.NewRegion(dev, testRefcountStart, testDataBlocks, nil) +func (env *workerEnv) write(ino *inode.Inode) { + env.t.Helper() + if err := env.tbl.Write(env.ctx, ino); err != nil { + env.t.Fatalf("write inode %d: %v", ino.InodeNum, err) + } +} - // Initialize refcount region - ctx := context.Background() - if err := refReg.Initialize(ctx); err != nil { - t.Fatalf("Failed to initialize refcount region: %v", err) +func (env *workerEnv) file(blocks uint64) (*inode.Inode, []uint64) { + env.t.Helper() + ino := env.claim() + if _, err := env.ext.AllocateBlocks(env.ctx, ino, 0, blocks); err != nil { + env.t.Fatal(err) + } + env.write(ino) + data, err := env.ext.DataBlocks(ino) + if err != nil { + env.t.Fatal(err) } + return ino, data +} - // Create snapshot registry - snapReg := NewRegistry(dev, testSnapshotStart, testSnapshotSize, nil) +// snapshot captures file the way createSnapshot does and leaves it in state. +func (env *workerEnv) snapshot(file *inode.Inode, name string, state State) *Descriptor { + env.t.Helper() + data := env.claim() + data.Flags |= inode.FlagSnapshotData + env.write(data) + desc, err := env.reg.Create(env.ctx, name, file.InodeNum, 0, data.InodeNum, file.AllocatedBlocks, file.Size, FlagReadOnly) + if err != nil { + env.t.Fatal(err) + } + blocks, err := env.ext.DataBlocks(file) + if err != nil { + env.t.Fatal(err) + } + if err := env.refs.Share(env.ctx, blocks); err != nil { + env.t.Fatal(err) + } + if err := env.ext.CopyExtentsForClone(env.ctx, file, data); err != nil { + env.t.Fatal(err) + } + env.write(data) + if err := env.ext.MarkAllShared(file); err != nil { + env.t.Fatal(err) + } + env.write(file) + if state != StateCreating { + if err := env.reg.UpdateState(env.ctx, desc.SnapshotID, state); err != nil { + env.t.Fatal(err) + } + } + return desc +} - // Initialize snapshot registry - if err := snapReg.Initialize(ctx); err != nil { - t.Fatalf("Failed to initialize snapshot registry: %v", err) +// remove marks a snapshot deleting, runs its delete job, and waits for it. +func (env *workerEnv) remove(id uint64) { + env.t.Helper() + if err := env.reg.Delete(env.ctx, id); err != nil { + env.t.Fatal(err) + } + if _, err := env.w.QueueDeleteJob(env.ctx, id); err != nil { + env.t.Fatal(err) } + env.wait(1) +} - // Create extent manager (bitmap can be nil for tests not needing allocation) - extMgr := extent.NewManager(dev, nil, nil, 0) +func (env *workerEnv) wait(jobs int) { + env.t.Helper() + timeout := time.After(10 * time.Second) + for range jobs { + select { + case r := <-env.done: + if !r.Success { + env.t.Fatalf("job %d for snapshot %d failed: %v", r.JobID, r.SnapshotID, r.Error) + } + case <-timeout: + env.t.Fatalf("timed out waiting for %d snapshot job(s)", jobs) + } + } +} - // Create inode table - var nodeUUID [16]byte - inodeTbl := inode.NewTable(dev, testInodeStart, testInodeSize, nil, nodeUUID) +func (env *workerEnv) allocated(block uint64) bool { + env.t.Helper() + ok, err := env.bm.IsAllocated(block) + if err != nil { + env.t.Fatal(err) + } + return ok +} - return snapReg, refReg, extMgr, inodeTbl +func (env *workerEnv) checkGone(desc *Descriptor) { + env.t.Helper() + if _, err := env.reg.Get(env.ctx, desc.SnapshotID); !errors.Is(err, ErrSnapshotNotFound) { + env.t.Errorf("snapshot %q still listed: %v", desc.GetName(), err) + } + if ino, err := env.tbl.Read(desc.DataInode); err != nil || ino.Magic != 0 { + env.t.Errorf("data inode %d of snapshot %q not freed (err=%v)", desc.DataInode, desc.GetName(), err) + } } func TestWorkerStartStop(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) + env := newWorkerEnv(t) + worker := env.w if worker.IsRunning() { t.Error("Worker should not be running before Start()") @@ -156,242 +255,75 @@ func TestWorkerStartStop(t *testing.T) { } } -func TestWorkerQueueCreateJob(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create a snapshot first - desc, err := snapReg.Create(ctx, "test-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - cfg.OnJobComplete = func(result *JobResult) { - t.Logf("Job %d completed: success=%v, duration=%v", result.JobID, result.Success, result.Duration) - } - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue a create job - blockNums := []uint64{10, 11, 12} - jobID, err := worker.QueueCreateJob(ctx, desc.SnapshotID, 100, blockNums) - if err != nil { - t.Fatalf("Failed to queue create job: %v", err) - } - - if jobID == 0 { - t.Error("Job ID should not be 0") - } - - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) - - // Verify refcounts were incremented - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 1 { - t.Errorf("Block %d refcount = %d, want 1", blockNum, entry.RefCount) - } - } - - // Verify snapshot state changed to Active - updatedDesc, err := snapReg.Get(ctx, desc.SnapshotID) - if err != nil { - t.Fatalf("Failed to get snapshot: %v", err) - } - if updatedDesc.State != StateActive { - t.Errorf("Snapshot state = %v, want StateActive", updatedDesc.State) - } -} - func TestWorkerQueueDeleteJob(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create and complete a snapshot first - desc, err := snapReg.Create(ctx, "test-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } - - // Set up initial refcounts - blockNums := []uint64{20, 21, 22} - for _, blockNum := range blockNums { - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount for block %d: %v", blockNum, err) - } - } - - // Mark snapshot as active, then delete - if err := snapReg.UpdateState(ctx, desc.SnapshotID, StateActive); err != nil { - t.Fatalf("Failed to update state: %v", err) - } - if err := snapReg.Delete(ctx, desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job - jobID, err := worker.QueueDeleteJob(ctx, desc.SnapshotID, blockNums) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - if jobID == 0 { - t.Error("Job ID should not be 0") - } - - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) - - // Verify refcounts were decremented - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 0 { - t.Errorf("Block %d refcount = %d, want 0", blockNum, entry.RefCount) - } - } + env := newWorkerEnv(t) + env.start() + initial := env.bm.FreeCount() - // Verify snapshot was removed - _, err = snapReg.Get(ctx, desc.SnapshotID) - if err != ErrSnapshotNotFound { - t.Errorf("Snapshot should be removed, got err = %v", err) - } -} - -func TestWorkerNotRunning(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() + file, blocks := env.file(3) + snap := env.snapshot(file, "test-snap", StateActive) - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - // Don't start the worker - - _, err := worker.QueueCreateJob(ctx, 1, 100, []uint64{1, 2, 3}) - if err == nil { - t.Error("QueueCreateJob should fail when worker not running") + // The file lets go of its last two blocks; only the snapshot holds them now. + if err := env.ext.Truncate(env.ctx, file, extent.BlockSize); err != nil { + t.Fatal(err) } - - _, err = worker.QueueDeleteJob(ctx, 1, []uint64{1, 2, 3}) - if err == nil { - t.Error("QueueDeleteJob should fail when worker not running") + env.write(file) + if !env.allocated(blocks[1]) || !env.allocated(blocks[2]) { + t.Fatal("truncating the file freed blocks the snapshot holds") } -} -func TestWorkerEmptyBlockList(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() - - // Create snapshot - desc, err := snapReg.Create(ctx, "empty-snap", 100, 0, 0, 0, 0, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue create job with empty block list - _, err = worker.QueueCreateJob(ctx, desc.SnapshotID, 100, []uint64{}) - if err != nil { - t.Fatalf("Failed to queue empty create job: %v", err) + env.remove(snap.SnapshotID) + env.checkGone(snap) + if env.allocated(blocks[1]) || env.allocated(blocks[2]) { + t.Error("blocks only the snapshot held stay allocated") } - - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) - - // Verify snapshot state changed to Active - updatedDesc, err := snapReg.Get(ctx, desc.SnapshotID) - if err != nil { - t.Fatalf("Failed to get snapshot: %v", err) + if !env.allocated(blocks[0]) { + t.Error("deleting the snapshot freed a block the file still maps") } - if updatedDesc.State != StateActive { - t.Errorf("Snapshot state = %v, want StateActive", updatedDesc.State) + if got := env.bm.FreeCount(); got != initial-1 { + t.Errorf("free blocks = %d, want %d", got, initial-1) } } // TestWorkerQueuesJobsWithCanceledContext checks that a canceled FUSE request // context cannot drop a job for a snapshot that already exists. func TestWorkerQueuesJobsWithCanceledContext(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, DefaultWorkerConfig()) - worker.Start() - defer worker.Stop() + env := newWorkerEnv(t) + file, _ := env.file(1) + var snaps []*Descriptor + for i := range 20 { + snaps = append(snaps, env.snapshot(file, fmt.Sprintf("snap%d", i), StateDeleting)) + } + env.start() - canceled, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(env.ctx) cancel() - - for i := 0; i < 20; i++ { - desc, err := snapReg.Create(context.Background(), fmt.Sprintf("snap-%d", i), 100, 0, 0, 0, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot %d: %v", i, err) + for _, snap := range snaps { + if _, err := env.w.QueueDeleteJob(ctx, snap.SnapshotID); err != nil { + t.Fatalf("queue delete of %q: %v", snap.GetName(), err) } + } + env.wait(len(snaps)) + for _, snap := range snaps { + env.checkGone(snap) + } +} - if _, err := worker.QueueCreateJob(canceled, desc.SnapshotID, 100, []uint64{}); err != nil { - t.Fatalf("QueueCreateJob %d with canceled context: %v", i, err) - } - waitForJob(t, snapshotActive(snapReg, desc.SnapshotID)) - - if err := snapReg.Delete(context.Background(), desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot %d: %v", i, err) - } - if _, err := worker.QueueDeleteJob(canceled, desc.SnapshotID, []uint64{}); err != nil { - t.Fatalf("QueueDeleteJob %d with canceled context: %v", i, err) - } - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) +func TestWorkerNotRunning(t *testing.T) { + env := newWorkerEnv(t) + if _, err := env.w.QueueDeleteJob(env.ctx, 1); !errors.Is(err, ErrWorkerStopped) { + t.Errorf("QueueDeleteJob on a stopped worker: got %v, want ErrWorkerStopped", err) } } func TestWorkerPendingJobs(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - - cfg := DefaultWorkerConfig() - cfg.QueueSize = 10 - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - - // Before starting, pending should be 0 - if worker.PendingJobs() != 0 { - t.Errorf("PendingJobs = %d before start, want 0", worker.PendingJobs()) + env := newWorkerEnv(t) + if env.w.PendingJobs() != 0 { + t.Errorf("PendingJobs = %d before start, want 0", env.w.PendingJobs()) } - - worker.Start() - defer worker.Stop() - - // After starting with no jobs, still 0 - if worker.PendingJobs() != 0 { - t.Errorf("PendingJobs = %d after start, want 0", worker.PendingJobs()) + env.start() + if env.w.PendingJobs() != 0 { + t.Errorf("PendingJobs = %d after start, want 0", env.w.PendingJobs()) } } @@ -406,83 +338,70 @@ func TestDefaultWorkerConfig(t *testing.T) { } } -// TestLeafSnapshotDeletion tests deleting a snapshot with no children. -// Expected behavior: -// - Refcounts decremented -// - Blocks with refcount=0 are freed via bitmap -// - Snapshot descriptor removed +// TestLeafSnapshotDeletion deletes the only owner left after the file is gone. func TestLeafSnapshotDeletion(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + env.start() + initial := env.bm.FreeCount() - // Create a snapshot (will be a leaf since it has no children) - desc, err := snapReg.Create(ctx, "leaf-snap", 100, 0, 0, 3, 1024, FlagReadOnly) - if err != nil { - t.Fatalf("Failed to create snapshot: %v", err) + file, blocks := env.file(2) + snap := env.snapshot(file, "leaf-snap", StateActive) + if err := env.ext.Delete(env.ctx, file); err != nil { + t.Fatal(err) } - - // Set up initial refcounts (simulating blocks used by snapshot) - blockNums := []uint64{30, 31, 32} - for _, blockNum := range blockNums { - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount for block %d: %v", blockNum, err) + if err := env.tbl.Free(env.ctx, file.InodeNum); err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if !env.allocated(b) { + t.Fatalf("deleting the file freed block %d that the snapshot holds", b) } } - // Mark as active then delete - if err := snapReg.UpdateState(ctx, desc.SnapshotID, StateActive); err != nil { - t.Fatalf("Failed to update state: %v", err) - } - if err := snapReg.Delete(ctx, desc.SnapshotID); err != nil { - t.Fatalf("Failed to delete snapshot: %v", err) + env.remove(snap.SnapshotID) + env.checkGone(snap) + if got := env.bm.FreeCount(); got != initial { + t.Errorf("free blocks = %d, want %d", got, initial) } +} - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() +func TestRecoverIncompleteSnapshots(t *testing.T) { + env := newWorkerEnv(t) + file, blocks := env.file(2) + active := env.snapshot(file, "active", StateActive) + creating := env.snapshot(file, "creating", StateCreating) + deleting := env.snapshot(file, "deleting", StateDeleting) + env.start() - // Queue delete job - _, err = worker.QueueDeleteJob(ctx, desc.SnapshotID, blockNums) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) + if err := env.w.RecoverIncompleteJobs(env.ctx); err != nil { + t.Fatal(err) } + env.wait(2) - waitForJob(t, snapshotRemoved(snapReg, desc.SnapshotID)) - - // Verify refcounts are 0 (all decremented) - for _, blockNum := range blockNums { - entry, err := refReg.Get(ctx, blockNum) + env.checkGone(creating) + env.checkGone(deleting) + if got, err := env.reg.Get(env.ctx, active.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("active snapshot after recovery: %+v, %v", got, err) + } + for _, b := range blocks { + e, err := env.refs.Get(env.ctx, b) if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue + t.Fatal(err) } - if entry.RefCount != 0 { - t.Errorf("Block %d refcount = %d, want 0", blockNum, entry.RefCount) + if !env.allocated(b) || e.RefCount != 2 { + t.Errorf("block %d: allocated=%v refcount=%d, want held by the file and one snapshot", b, env.allocated(b), e.RefCount) } } - - // Verify snapshot is removed - _, err = snapReg.Get(ctx, desc.SnapshotID) - if err != ErrSnapshotNotFound { - t.Errorf("Expected ErrSnapshotNotFound, got %v", err) - } } // TestMiddleSnapshotDeletionMergeUp tests deleting a snapshot that has children. // Expected behavior: // - Children are re-parented to grandparent (merge-up) -// - Refcounts decremented // - Snapshot descriptor removed func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx + env.start() // Create parent snapshot (root) rootSnap, err := snapReg.Create(ctx, "root-snap", 100, 0, 0, 2, 1024, FlagReadOnly) @@ -511,35 +430,7 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { t.Fatalf("Failed to activate leaf: %v", err) } - // Set up refcounts for middle snapshot's blocks - middleBlocks := []uint64{40, 41, 42} - for _, blockNum := range middleBlocks { - // Increment twice: once for middle, once simulating shared with leaf - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount: %v", err) - } - if _, err := refReg.Increment(ctx, blockNum); err != nil { - t.Fatalf("Failed to increment refcount: %v", err) - } - } - - // Delete middle snapshot - if err := snapReg.Delete(ctx, middleSnap.SnapshotID); err != nil { - t.Fatalf("Failed to delete middle snapshot: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job for middle snapshot - _, err = worker.QueueDeleteJob(ctx, middleSnap.SnapshotID, middleBlocks) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - waitForJob(t, snapshotRemoved(snapReg, middleSnap.SnapshotID)) + env.remove(middleSnap.SnapshotID) // Verify leaf snapshot was re-parented to root updatedLeaf, err := snapReg.Get(ctx, leafSnap.SnapshotID) @@ -551,18 +442,6 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { updatedLeaf.ParentID, rootSnap.SnapshotID) } - // Verify refcounts decremented (should be 1 now, not 0, due to leaf sharing) - for _, blockNum := range middleBlocks { - entry, err := refReg.Get(ctx, blockNum) - if err != nil { - t.Errorf("Failed to get refcount for block %d: %v", blockNum, err) - continue - } - if entry.RefCount != 1 { - t.Errorf("Block %d refcount = %d, want 1 (shared with leaf)", blockNum, entry.RefCount) - } - } - // Verify middle snapshot is removed _, err = snapReg.Get(ctx, middleSnap.SnapshotID) if err != ErrSnapshotNotFound { @@ -582,11 +461,8 @@ func TestMiddleSnapshotDeletionMergeUp(t *testing.T) { // TestGetChildren verifies the GetChildren registry method. func TestGetChildren(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, _, _, _ := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx // Create parent snapshot parent, err := snapReg.Create(ctx, "parent", 100, 0, 0, 1, 100, FlagReadOnly) @@ -646,11 +522,8 @@ func TestGetChildren(t *testing.T) { // TestUpdateParentID verifies the UpdateParentID registry method. func TestUpdateParentID(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, _, _, _ := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx // Create a chain: grandparent -> parent -> child grandparent, err := snapReg.Create(ctx, "grandparent", 100, 0, 0, 1, 100, FlagReadOnly) @@ -697,11 +570,9 @@ func TestUpdateParentID(t *testing.T) { // TestMultipleChildrenMergeUp tests merge-up when a snapshot has multiple children. func TestMultipleChildrenMergeUp(t *testing.T) { - dev, _, cleanup := setupTestDevice(t) - defer cleanup() - - snapReg, refReg, extMgr, inodeTbl := setupTestComponents(t, dev) - ctx := context.Background() + env := newWorkerEnv(t) + snapReg, ctx := env.reg, env.ctx + env.start() // Create: root -> middle -> {child1, child2, child3} root, err := snapReg.Create(ctx, "root", 100, 0, 0, 1, 100, FlagReadOnly) @@ -732,23 +603,7 @@ func TestMultipleChildrenMergeUp(t *testing.T) { children = append(children, child) } - // Delete middle - if err := snapReg.Delete(ctx, middle.SnapshotID); err != nil { - t.Fatalf("Failed to delete middle: %v", err) - } - - cfg := DefaultWorkerConfig() - worker := NewWorker(snapReg, refReg, extMgr, inodeTbl, cfg) - worker.Start() - defer worker.Stop() - - // Queue delete job - _, err = worker.QueueDeleteJob(ctx, middle.SnapshotID, []uint64{}) - if err != nil { - t.Fatalf("Failed to queue delete job: %v", err) - } - - waitForJob(t, snapshotRemoved(snapReg, middle.SnapshotID)) + env.remove(middle.SnapshotID) // Verify all children now point to root for i, child := range children { From 8364b7ddfb0f237c2d2c828ac54042a27802e7a6 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:26:47 -0500 Subject: [PATCH 04/18] test(snapshot): check snapshot data, clone writes and delete in all suites Adds Go, Docker and VM tests that read a snapshot through a clone after in-place overwrite, append and truncate, write into a clone and its source with dd conv=notrunc, and delete a snapshot after an overwrite. The Docker image and VMs install attr for setfattr. Co-Authored-By: Claude Opus 5.5 --- tests/docker/Dockerfile | 2 + tests/docker/scripts/run-tests.sh | 138 ++++++++++++++++++++++++++ tests/integration/extent_tree_test.go | 17 ++-- tests/integration/snapshot_test.go | 129 ++++++++++++++++++++++++ tests/test-manifest.yaml | 67 +++++++++++-- tests/vm/scripts/build-vm-image.sh | 2 +- tests/vm/scripts/run-vm-tests.sh | 134 +++++++++++++++++++++++++ tests/vm/vm-test-harness.sh | 6 +- 8 files changed, 479 insertions(+), 16 deletions(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index e75321e2..6c71e0f1 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -24,11 +24,13 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o goclusterfs-mount ./cmd/mount FROM debian:bookworm-slim AS runtime # Install runtime dependencies: +# - attr: setfattr/getfattr for snapshot and clone tests # - fuse3: FUSE filesystem support # - iproute2: tc (traffic control) for network simulation # - iptables: Firewall rules for network partition simulation # - procps: ps, top for debugging RUN apt-get update && apt-get install -y --no-install-recommends \ + attr \ fuse3 \ iproute2 \ iptables \ diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index adca04ae..21e44aec 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2796,6 +2796,141 @@ test_extent_tree_interleaved_appends() { return 0 } +md5_on_node_matches() { + local node=$1 path=$2 want=$3 got + got=$(exec_on_node "$node" sh -c "md5sum < $path | cut -d' ' -f1") + if [ "$got" != "$want" ]; then + log "ERROR: $node $path md5 $got, want $want" + return 1 + fi +} + +# Snapshots a file, then overwrites it in place, appends to it and truncates +# it. A clone of the snapshot must hold the original data on both nodes. The +# expected data is built with the same edits on a local copy. +test_snapshot_preserves_data() { + log "Testing that a snapshot keeps its data..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]:-${NODE_ARRAY[0]}}" + local file=/mnt/goclusterfs/snap_data.bin + local clone=/mnt/goclusterfs/snap_data_clone.bin + + exec_on_node "$writer" rm -f "$file" "$clone" + exec_on_node "$writer" sh -c "head -c 3145728 /dev/urandom > /tmp/snap_orig && cp /tmp/snap_orig /tmp/snap_live && cp /tmp/snap_orig $file" || return 1 + exec_on_node "$writer" setfattr -n user.snapshot -v snap_data_before "$file" || { + log "ERROR: snapshot failed" + return 1 + } + exec_on_node "$writer" sh -c "head -c 4096 /dev/urandom > /tmp/snap_patch && head -c 2097152 /dev/urandom > /tmp/snap_tail" + for target in "$file" /tmp/snap_live; do + exec_on_node "$writer" sh -c "dd if=/tmp/snap_patch of=$target bs=4096 seek=257 conv=notrunc 2>/dev/null && cat /tmp/snap_tail >> $target && truncate -s 1572864 $target" || { + log "ERROR: changing $target failed" + return 1 + } + done + exec_on_node "$writer" setfattr -n user.snapshot.clone -v "snap_data_before:/snap_data_clone.bin" "$file" || { + log "ERROR: clone from snapshot failed" + return 1 + } + + local want_clone want_live + want_clone=$(exec_on_node "$writer" sh -c "md5sum < /tmp/snap_orig | cut -d' ' -f1") + want_live=$(exec_on_node "$writer" sh -c "md5sum < /tmp/snap_live | cut -d' ' -f1") + for node in $(printf '%s\n' "$reader" "$writer" | uniq); do + restart_node "$node" + wait_node_healthy "$node" 30 || return 1 + md5_on_node_matches "$node" "$clone" "$want_clone" || return 1 + md5_on_node_matches "$node" "$file" "$want_live" || return 1 + done + + exec_on_node "$writer" setfattr -x user.snapshot.snap_data_before "$file" + log "Snapshot keeps its data test passed" + return 0 +} + +# Writes into a clone and into its source with dd conv=notrunc. Each write must +# change only the file written, on both nodes. +test_clone_in_place_write() { + log "Testing in-place writes to a clone and its source..." + + local writer="${NODE_ARRAY[0]}" + local reader="${NODE_ARRAY[1]:-${NODE_ARRAY[0]}}" + local src=/mnt/goclusterfs/clone_src.bin + local clone=/mnt/goclusterfs/clone_inplace.bin + + exec_on_node "$writer" rm -f "$src" "$clone" + exec_on_node "$writer" sh -c "head -c 2097152 /dev/urandom > /tmp/clone_src && cp /tmp/clone_src /tmp/clone_copy && cp /tmp/clone_src $src" || return 1 + exec_on_node "$writer" setfattr -n user.clone -v /clone_inplace.bin "$src" || { + log "ERROR: clone failed" + return 1 + } + exec_on_node "$writer" sh -c "head -c 4096 /dev/urandom > /tmp/clone_patch_a && head -c 4096 /dev/urandom > /tmp/clone_patch_b" + exec_on_node "$writer" sh -c "dd if=/tmp/clone_patch_a of=$clone bs=4096 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_a of=/tmp/clone_copy bs=4096 conv=notrunc 2>/dev/null" || return 1 + exec_on_node "$writer" sh -c "dd if=/tmp/clone_patch_b of=$src bs=4096 seek=256 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_b of=/tmp/clone_src bs=4096 seek=256 conv=notrunc 2>/dev/null" || return 1 + + local want_src want_clone + want_src=$(exec_on_node "$writer" sh -c "md5sum < /tmp/clone_src | cut -d' ' -f1") + want_clone=$(exec_on_node "$writer" sh -c "md5sum < /tmp/clone_copy | cut -d' ' -f1") + for node in $(printf '%s\n' "$reader" "$writer" | uniq); do + restart_node "$node" + wait_node_healthy "$node" 30 || return 1 + md5_on_node_matches "$node" "$src" "$want_src" || return 1 + md5_on_node_matches "$node" "$clone" "$want_clone" || return 1 + done + + log "In-place clone write test passed" + return 0 +} + +# Snapshots a file, overwrites both of its blocks, then deletes the snapshot. +# The delete must finish and give back exactly the two blocks only the +# snapshot held. +test_snapshot_delete_after_overwrite() { + log "Testing snapshot delete after overwrite..." + + local node="${NODE_ARRAY[0]}" + local file=/mnt/goclusterfs/snap_delete.bin + local free_before free_now + + exec_on_node "$node" rm -f "$file" + exec_on_node "$node" sh -c "head -c 2097152 /dev/urandom > $file" || return 1 + free_before=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + exec_on_node "$node" setfattr -n user.snapshot -v snap_delete_temp "$file" || { + log "ERROR: snapshot failed" + return 1 + } + exec_on_node "$node" sh -c "head -c 2097152 /dev/urandom > /tmp/snap_new && dd if=/tmp/snap_new of=$file bs=1048576 conv=notrunc 2>/dev/null" || return 1 + free_now=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + if [ "$free_now" -ne $((free_before - 2)) ]; then + log "ERROR: $free_now free blocks after overwriting 2 snapshotted blocks, want $((free_before - 2))" + return 1 + fi + + exec_on_node "$node" setfattr -x user.snapshot.snap_delete_temp "$file" || { + log "ERROR: snapshot delete failed" + return 1 + } + local tries=0 + while exec_on_node "$node" sh -c "getfattr --only-values -n user.snapshots $file 2>/dev/null | grep -q snap_delete_temp"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log "ERROR: snapshot still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + free_now=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) + if [ "$free_now" -ne "$free_before" ]; then + log "ERROR: $free_now free blocks after deleting the snapshot, want $free_before" + return 1 + fi + md5_on_node_matches "$node" "$file" "$(exec_on_node "$node" sh -c "md5sum < /tmp/snap_new | cut -d' ' -f1")" || return 1 + + log "Snapshot delete after overwrite test passed" + return 0 +} + run_test "Create and Read File" test_create_read_file run_test "Create Directory" test_create_directory run_test "File in Directory" test_file_in_directory @@ -2813,6 +2948,9 @@ run_test "Overwrite File" test_overwrite_file run_test "Sequential Access" test_sequential_access run_test "Extent Tree Sparse Writes" test_extent_tree_sparse_writes run_test "Extent Tree Interleaved Appends" test_extent_tree_interleaved_appends +run_test "Snapshot Preserves Data" test_snapshot_preserves_data +run_test "Clone In-Place Write" test_clone_in_place_write +run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite if [ "$SKIP_CONCURRENT_TESTS" = "true" ]; then log "SKIPPING: Concurrent Writes (known race conditions - set SKIP_CONCURRENT_TESTS=false to enable)" diff --git a/tests/integration/extent_tree_test.go b/tests/integration/extent_tree_test.go index ede43b08..d4772ab7 100644 --- a/tests/integration/extent_tree_test.go +++ b/tests/integration/extent_tree_test.go @@ -220,12 +220,10 @@ func TestExtentTreeSnapshot(t *testing.T) { } t.Fatalf("Failed to create snapshot: %v", err) } - time.Sleep(500 * time.Millisecond) // background refcount job if err := unix.Setxattr(path, "user.snapshot.clone", []byte("tree-snap:/clone.bin"), 0); err != nil { t.Fatalf("clone from snapshot failed: %v", err) } - time.Sleep(500 * time.Millisecond) cloneWant := append([]byte(nil), want...) checkFile(t, mfs, "clone.bin", cloneWant) @@ -248,7 +246,7 @@ func TestExtentTreeSnapshot(t *testing.T) { f.Close() want = append(want, tail...) - // Blocks 4 and 5 are shared with the clone; only the new block 6 is freed. + // Blocks 4 and 5 are shared with the clone and the snapshot; only the new block 6 is freed. before := freeBlocks() if err := os.Truncate(path, 4*mb); err != nil { t.Fatalf("Truncate failed: %v", err) @@ -264,9 +262,16 @@ func TestExtentTreeSnapshot(t *testing.T) { if err := unix.Removexattr(path, "user.snapshot.tree-snap"); err != nil && err != syscall.ENODATA { t.Fatalf("deleting snapshot failed: %v", err) } - time.Sleep(500 * time.Millisecond) - if got := freeBlocks() - before; got != 0 { - t.Errorf("deleting the snapshot freed %d blocks, want 0", got) + listed := make([]byte, 4096) + if _, ok := waitFor(10*time.Second, func() bool { + n, err := unix.Getxattr(path, "user.snapshots", listed) + return err == nil && !bytes.Contains(listed[:n], []byte("tree-snap")) + }); !ok { + t.Fatal("snapshot still listed 10s after delete") + } + // The snapshot's own extent tree node is the only block nothing else maps. + if got := freeBlocks() - before; got != 1 { + t.Errorf("deleting the snapshot freed %d blocks, want 1", got) } remount(t, mfs) diff --git a/tests/integration/snapshot_test.go b/tests/integration/snapshot_test.go index 2ec912e4..c8941c6c 100644 --- a/tests/integration/snapshot_test.go +++ b/tests/integration/snapshot_test.go @@ -823,3 +823,132 @@ func TestCloneFromSnapshot(t *testing.T) { t.Logf("Clone content: %q", cloneContent) t.Log("Clone from snapshot test passed") } + +func setxattr(t *testing.T, path, attr, value string) { + t.Helper() + if err := unix.Setxattr(path, attr, []byte(value), 0); err != nil { + t.Fatalf("setxattr %s=%q on %s: %v", attr, value, path, err) + } +} + +func overwrite(base []byte, off int, data []byte) []byte { + out := bytes.Clone(base) + copy(out[off:], data) + return out +} + +func freeBlocks(t *testing.T, mfs *MountedFS) uint64 { + t.Helper() + var st unix.Statfs_t + if err := unix.Statfs(mfs.Path(""), &st); err != nil { + t.Fatalf("statfs: %v", err) + } + return st.Bfree +} + +// TestSnapshotPreservesData reads a snapshot, through a clone of it, after the +// file was overwritten in place, appended to and truncated. +func TestSnapshotPreservesData(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + writer, reader := cluster.Node(0), cluster.Node(1) + + original := chunk(1, 3*mb) + if err := writer.WriteFile("data.bin", original); err != nil { + t.Fatal(err) + } + checkFile(t, writer, "data.bin", original) + setxattr(t, writer.Path("data.bin"), "user.snapshot", "before") + + if err := writer.WriteFileAt("data.bin", chunk(2, 4096), mb+100); err != nil { + t.Fatal(err) + } + if err := writer.WriteFileAt("data.bin", chunk(3, 2*mb), 3*mb); err != nil { + t.Fatal(err) + } + if err := os.Truncate(writer.Path("data.bin"), mb+mb/2); err != nil { + t.Fatal(err) + } + setxattr(t, writer.Path("data.bin"), "user.snapshot.clone", "before:/before-clone") + + live := overwrite(original[:mb+mb/2], mb+100, chunk(2, 4096)) + for _, node := range []*MountedFS{writer, reader} { + checkFile(t, node, "before-clone", original) + checkFile(t, node, "data.bin", live) + } +} + +// TestCloneInPlaceWrite writes into a clone and its source without truncating, +// after both were read, so the reads that follow must not reuse cached blocks. +func TestCloneInPlaceWrite(t *testing.T) { + mfs := NewMountedFS(t, 20) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + original := chunk(4, 2*mb) + if err := mfs.WriteFile("source.bin", original); err != nil { + t.Fatal(err) + } + setxattr(t, mfs.Path("source.bin"), "user.clone", "/clone.bin") + checkFile(t, mfs, "source.bin", original) + checkFile(t, mfs, "clone.bin", original) + + if err := mfs.WriteFileAt("clone.bin", chunk(5, 100), 10); err != nil { + t.Fatal(err) + } + if err := mfs.WriteFileAt("source.bin", chunk(6, 100), mb+10); err != nil { + t.Fatal(err) + } + + wantSource := overwrite(original, mb+10, chunk(6, 100)) + wantClone := overwrite(original, 10, chunk(5, 100)) + checkFile(t, mfs, "source.bin", wantSource) + checkFile(t, mfs, "clone.bin", wantClone) + remount(t, mfs) + checkFile(t, mfs, "source.bin", wantSource) + checkFile(t, mfs, "clone.bin", wantClone) +} + +// TestSnapshotDeleteAfterOverwrite deletes a snapshot whose blocks the file no +// longer maps; the delete must finish and give back exactly those blocks. +func TestSnapshotDeleteAfterOverwrite(t *testing.T) { + mfs := NewMountedFS(t, 20) + if err := mfs.FormatAndMount(false); err != nil { + t.Fatalf("FormatAndMount failed: %v", err) + } + + if err := mfs.WriteFile("data.bin", chunk(7, 2*mb)); err != nil { + t.Fatal(err) + } + free := freeBlocks(t, mfs) + setxattr(t, mfs.Path("data.bin"), "user.snapshot", "temp") + + updated := chunk(8, 2*mb) + if err := mfs.WriteFileAt("data.bin", updated, 0); err != nil { + t.Fatal(err) + } + if got := freeBlocks(t, mfs); got != free-2 { + t.Fatalf("free blocks after overwriting 2 snapshotted blocks = %d, want %d", got, free-2) + } + + if err := unix.Removexattr(mfs.Path("data.bin"), "user.snapshot.temp"); err != nil { + t.Fatalf("delete snapshot: %v", err) + } + listed := make([]byte, 4096) + if _, ok := waitFor(10*time.Second, func() bool { + n, err := unix.Getxattr(mfs.Path("data.bin"), "user.snapshots", listed) + return err == nil && !bytes.Contains(listed[:n], []byte("temp")) + }); !ok { + t.Fatal("snapshot still listed 10s after delete") + } + if got := freeBlocks(t, mfs); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, mfs, "data.bin", updated) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index eb9c88af..c72d15c9 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -650,7 +650,7 @@ scenarios: - id: extent-tree-snapshot name: Snapshot and clone of a fragmented file - description: A clone of a file with more than 2 extents gets its own extent tree; blocks it shares survive truncating and deleting the source, and deleting the snapshot frees nothing in use + description: A clone of a file with more than 2 extents gets its own extent tree; blocks it shares survive truncating and deleting the source, and deleting the snapshot frees only its own extent tree node priority: high # Docker/VM TODO: snapshot and clone tests need xattr support in those harnesses, like the other snapshot scenarios. environments: @@ -1287,20 +1287,48 @@ scenarios: environments: go: internal/snapshot/worker_test.go::TestWorkerStartStop - - id: snapshot-worker-create-job - name: Snapshot create job processing - description: Background worker processes refcount increment jobs + - id: snapshot-worker-recovery + name: Snapshot recovery after a crash + description: On mount, snapshots left creating or deleting give back their captured blocks and disappear, without touching blocks the live file still maps priority: go_only environments: - go: internal/snapshot/worker_test.go::TestWorkerQueueCreateJob + go: internal/snapshot/worker_test.go::TestRecoverIncompleteSnapshots - id: snapshot-worker-delete-job name: Snapshot delete job processing - description: Background worker processes refcount decrement jobs + description: Background worker frees the snapshot's own blocks, keeps blocks the live file still maps, and removes the descriptor priority: go_only environments: go: internal/snapshot/worker_test.go::TestWorkerQueueDeleteJob + - id: snapshot-queue-canceled-context + name: Snapshot delete queued from a canceled request + description: A delete job queued with an already canceled request context still runs + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestQueueDeleteJobIgnoresCanceledContext + + - id: refcount-share-release + name: Block reference sharing + description: Sharing a block counts the existing owner, and releasing reports a block as free only when its last owner lets go + priority: go_only + environments: + go: internal/refcount/refcount_test.go::TestShareAndRelease + + - id: extent-free-shared-blocks + name: Freeing shared blocks + description: Truncating one of two files that share a block keeps the block allocated until the other file lets go + priority: go_only + environments: + go: internal/extent/extent_test.go::TestFreeKeepsSharedBlocks + + - id: snapshot-cow-in-process + name: Snapshot and clone copy-on-write + description: Snapshots keep their data through in-place writes, appends and truncates; clone writes never reach the source; deleting a snapshot after an overwrite frees exactly its blocks + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotKeepsDataThroughWrites + # Task #4712: Snapshot deletion (leaf + merge-up) - id: snapshot-leaf-deletion name: Leaf snapshot deletion @@ -1397,6 +1425,33 @@ scenarios: docker: TODO vm: TODO + - id: snapshot-preserves-data + name: Snapshot keeps its data + description: After a snapshot, in-place overwrite, append and truncate of the file; a clone of the snapshot read on another node holds the original data + priority: critical + environments: + go: snapshot_test.go::TestSnapshotPreservesData + docker: run-tests.sh::test_snapshot_preserves_data + vm: run-vm-tests.sh::test_snapshot_preserves_data + + - id: clone-in-place-write + name: In-place write to a clone + description: dd conv=notrunc into a clone, and into its source, changes only the file written + priority: critical + environments: + go: snapshot_test.go::TestCloneInPlaceWrite + docker: run-tests.sh::test_clone_in_place_write + vm: run-vm-tests.sh::test_clone_in_place_write + + - id: snapshot-delete-after-overwrite + name: Snapshot delete after overwrite + description: Deleting a snapshot after the file was overwritten finishes, frees the snapshot's blocks, and leaves the file intact + priority: high + environments: + go: snapshot_test.go::TestSnapshotDeleteAfterOverwrite + docker: run-tests.sh::test_snapshot_delete_after_overwrite + vm: run-vm-tests.sh::test_snapshot_delete_after_overwrite + - id: clone-from-snapshot name: Clone from snapshot description: Create writable clone from named snapshot via xattr diff --git a/tests/vm/scripts/build-vm-image.sh b/tests/vm/scripts/build-vm-image.sh index b66249be..f7f4c898 100755 --- a/tests/vm/scripts/build-vm-image.sh +++ b/tests/vm/scripts/build-vm-image.sh @@ -121,7 +121,7 @@ EOF log_info "Installing packages..." sudo chroot "${MOUNT_POINT}" /bin/sh -c " apk update - apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs + apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs attr # Configure SSH ssh-keygen -A diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 25df7901..7363633c 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2860,6 +2860,137 @@ test_extent_tree_sparse_writes() { return 0 } +md5_on_vm_matches() { + local vm_id=$1 path=$2 want=$3 got + got=$(ssh_run "$vm_id" "md5sum < $path | cut -d' ' -f1") + if [ "$got" != "$want" ]; then + log_error "VM $vm_id $path md5 $got, want $want" + return 1 + fi +} + +# Snapshots a file, then overwrites it in place, appends to it and truncates +# it. A clone of the snapshot must hold the original data on both VMs. The +# expected data is built with the same edits on a local copy. +test_snapshot_preserves_data() { + log_info "Testing that a snapshot keeps its data..." + ensure_cluster_healthy 2 + + local file=/mnt/goclusterfs/snap_data.bin + local clone=/mnt/goclusterfs/snap_data_clone.bin + + ssh_run 1 "rm -f $file $clone" + ssh_run 1 "head -c 3145728 /dev/urandom > /tmp/snap_orig && cp /tmp/snap_orig /tmp/snap_live && cp /tmp/snap_orig $file" || return 1 + ssh_run 1 "setfattr -n user.snapshot -v snap_data_before $file" || { + log_error "snapshot failed" + return 1 + } + ssh_run 1 "head -c 4096 /dev/urandom > /tmp/snap_patch && head -c 2097152 /dev/urandom > /tmp/snap_tail" + for target in "$file" /tmp/snap_live; do + ssh_run 1 "dd if=/tmp/snap_patch of=$target bs=4096 seek=257 conv=notrunc 2>/dev/null && cat /tmp/snap_tail >> $target && truncate -s 1572864 $target" || { + log_error "changing $target failed" + return 1 + } + done + ssh_run 1 "setfattr -n user.snapshot.clone -v snap_data_before:/snap_data_clone.bin $file" || { + log_error "clone from snapshot failed" + return 1 + } + + local want_clone want_live + want_clone=$(ssh_run 1 "md5sum < /tmp/snap_orig | cut -d' ' -f1") + want_live=$(ssh_run 1 "md5sum < /tmp/snap_live | cut -d' ' -f1") + for vm_id in 2 1; do + restart_goclusterfs_vm "$vm_id" || return 1 + md5_on_vm_matches "$vm_id" "$clone" "$want_clone" || return 1 + md5_on_vm_matches "$vm_id" "$file" "$want_live" || return 1 + done + + ssh_run 1 "setfattr -x user.snapshot.snap_data_before $file" + log_info "Snapshot keeps its data test passed" + return 0 +} + +# Writes into a clone and into its source with dd conv=notrunc. Each write must +# change only the file written, on both VMs. +test_clone_in_place_write() { + log_info "Testing in-place writes to a clone and its source..." + ensure_cluster_healthy 2 + + local src=/mnt/goclusterfs/clone_src.bin + local clone=/mnt/goclusterfs/clone_inplace.bin + + ssh_run 1 "rm -f $src $clone" + ssh_run 1 "head -c 2097152 /dev/urandom > /tmp/clone_src && cp /tmp/clone_src /tmp/clone_copy && cp /tmp/clone_src $src" || return 1 + ssh_run 1 "setfattr -n user.clone -v /clone_inplace.bin $src" || { + log_error "clone failed" + return 1 + } + ssh_run 1 "head -c 4096 /dev/urandom > /tmp/clone_patch_a && head -c 4096 /dev/urandom > /tmp/clone_patch_b" + ssh_run 1 "dd if=/tmp/clone_patch_a of=$clone bs=4096 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_a of=/tmp/clone_copy bs=4096 conv=notrunc 2>/dev/null" || return 1 + ssh_run 1 "dd if=/tmp/clone_patch_b of=$src bs=4096 seek=256 conv=notrunc 2>/dev/null && dd if=/tmp/clone_patch_b of=/tmp/clone_src bs=4096 seek=256 conv=notrunc 2>/dev/null" || return 1 + + local want_src want_clone + want_src=$(ssh_run 1 "md5sum < /tmp/clone_src | cut -d' ' -f1") + want_clone=$(ssh_run 1 "md5sum < /tmp/clone_copy | cut -d' ' -f1") + for vm_id in 2 1; do + restart_goclusterfs_vm "$vm_id" || return 1 + md5_on_vm_matches "$vm_id" "$src" "$want_src" || return 1 + md5_on_vm_matches "$vm_id" "$clone" "$want_clone" || return 1 + done + + log_info "In-place clone write test passed" + return 0 +} + +# Snapshots a file, overwrites both of its blocks, then deletes the snapshot. +# The delete must finish and give back exactly the two blocks only the +# snapshot held. +test_snapshot_delete_after_overwrite() { + log_info "Testing snapshot delete after overwrite..." + ensure_cluster_healthy 2 + + local file=/mnt/goclusterfs/snap_delete.bin + local free_before free_now + + ssh_run 1 "rm -f $file" + ssh_run 1 "head -c 2097152 /dev/urandom > $file" || return 1 + free_before=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + ssh_run 1 "setfattr -n user.snapshot -v snap_delete_temp $file" || { + log_error "snapshot failed" + return 1 + } + ssh_run 1 "head -c 2097152 /dev/urandom > /tmp/snap_new && dd if=/tmp/snap_new of=$file bs=1048576 conv=notrunc 2>/dev/null" || return 1 + free_now=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + if [ "$free_now" -ne $((free_before - 2)) ]; then + log_error "$free_now free blocks after overwriting 2 snapshotted blocks, want $((free_before - 2))" + return 1 + fi + + ssh_run 1 "setfattr -x user.snapshot.snap_delete_temp $file" || { + log_error "snapshot delete failed" + return 1 + } + local tries=0 + while ssh_run 1 "getfattr --only-values -n user.snapshots $file 2>/dev/null | grep -q snap_delete_temp"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log_error "snapshot still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + free_now=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") + if [ "$free_now" -ne "$free_before" ]; then + log_error "$free_now free blocks after deleting the snapshot, want $free_before" + return 1 + fi + md5_on_vm_matches 1 "$file" "$(ssh_run 1 "md5sum < /tmp/snap_new | cut -d' ' -f1")" || return 1 + + log_info "Snapshot delete after overwrite test passed" + return 0 +} + # Two VMs append 1MB alternately to their own file, so both files get # non-contiguous blocks and grow past the old 2MB limit. Checks both files on # each VM after restarting it. @@ -2961,6 +3092,9 @@ main() { run_test "Sequential Access" test_sequential_access run_test "Extent Tree Sparse Writes" test_extent_tree_sparse_writes run_test "Extent Tree Interleaved Appends" test_extent_tree_interleaved_appends + run_test "Snapshot Preserves Data" test_snapshot_preserves_data + run_test "Clone In-Place Write" test_clone_in_place_write + run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite run_test "Concurrent Writes" test_concurrent_writes run_test "Concurrent File Creation" test_concurrent_file_creation diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 34d68c4e..b7c26bd1 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -361,9 +361,9 @@ setup_goclusterfs() { "${PROJECT_ROOT}/goclusterfs-mount" \ "root@127.0.0.1:/usr/local/bin/goclusterfs-mount" - # Install fuse3 and gcompat (glibc compatibility for binary compiled on glibc systems) - log_info "Installing fuse3 and gcompat..." - ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat" + # Install fuse3, gcompat (glibc compatibility for binary compiled on glibc systems) and attr (setfattr for snapshot tests) + log_info "Installing fuse3, gcompat and attr..." + ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat attr" # Load FUSE kernel module (required before mounting any FUSE filesystem) log_info "Loading FUSE kernel module..." From b003b70598b44d52d1d3031cf77c170c31b226a9 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 11:28:15 -0500 Subject: [PATCH 05/18] fix(snapshot): free a snapshot's data inode after its descriptor is gone If a crash hit after the data inode was freed but before the descriptor was removed, another snapshot could reuse the slot and recovery would empty it. Freeing last leaks at most an empty inode, which fsck reports. Co-Authored-By: Claude Opus 5.5 --- internal/snapshot/worker.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index 926ef824..7062cba9 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -222,13 +222,21 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { return fmt.Errorf("failed to remove snapshot: %w", err) } + // Freed only once no descriptor names it, so a reused slot is never released again. + if snap.HasData() { + if err := w.inodeTable.Free(ctx, snap.DataInode); err != nil { + log.Printf("[snapshot/worker] Failed to free data inode %d of snapshot %d: %v", + snap.DataInode, job.SnapshotID, err) + } + } + log.Printf("[snapshot/worker] Snapshot %d deleted", job.SnapshotID) return nil } -// ReleaseData gives back the blocks held by a snapshot's data inode and frees -// the inode. The empty extent map is committed before any block is released, -// so a crash leaks blocks at worst and calling it again is safe. +// ReleaseData gives back the blocks held by a snapshot's data inode. The empty +// extent map is committed before any block is released, so a crash leaks +// blocks at worst and calling it again is safe. func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { const attempts = 5 for attempt := 1; ; attempt++ { @@ -237,7 +245,6 @@ func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { return err } if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { - // Already freed, and possibly reused by a regular file since. return nil } if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil { @@ -246,14 +253,13 @@ func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { } err = w.inodeTable.Write(ctx, ino) if err == nil { - break + return nil } w.extentMgr.DiscardPending(ctx, ino) if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts { return err } } - return w.inodeTable.Free(ctx, num) } // resultHandler processes completed job results. From 4b950803383f7e10941dc81bb0a22f3734977d61 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 12:54:59 -0500 Subject: [PATCH 06/18] test(snapshot): wait for deleted snapshots to disappear TestSnapshotDeleteAndReuse kept a fixed wait because the delete hit a refcount underflow and never finished; it now polls for removal like the other snapshot tests. The manifest names #14's canceled-context test. Co-Authored-By: Claude Opus 5.5 --- tests/integration/extent_tree_test.go | 9 +-------- tests/integration/snapshot_test.go | 12 ++---------- tests/test-manifest.yaml | 2 +- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/tests/integration/extent_tree_test.go b/tests/integration/extent_tree_test.go index d4772ab7..83fec21a 100644 --- a/tests/integration/extent_tree_test.go +++ b/tests/integration/extent_tree_test.go @@ -5,7 +5,6 @@ import ( "os" "syscall" "testing" - "time" "golang.org/x/sys/unix" ) @@ -262,13 +261,7 @@ func TestExtentTreeSnapshot(t *testing.T) { if err := unix.Removexattr(path, "user.snapshot.tree-snap"); err != nil && err != syscall.ENODATA { t.Fatalf("deleting snapshot failed: %v", err) } - listed := make([]byte, 4096) - if _, ok := waitFor(10*time.Second, func() bool { - n, err := unix.Getxattr(path, "user.snapshots", listed) - return err == nil && !bytes.Contains(listed[:n], []byte("tree-snap")) - }); !ok { - t.Fatal("snapshot still listed 10s after delete") - } + waitForSnapshotState(t, path, "", "tree-snap") // The snapshot's own extent tree node is the only block nothing else maps. if got := freeBlocks() - before; got != 1 { t.Errorf("deleting the snapshot freed %d blocks, want 1", got) diff --git a/tests/integration/snapshot_test.go b/tests/integration/snapshot_test.go index c8941c6c..21530bd5 100644 --- a/tests/integration/snapshot_test.go +++ b/tests/integration/snapshot_test.go @@ -504,9 +504,7 @@ func TestSnapshotDeleteAndReuse(t *testing.T) { if err != nil && err != syscall.ENODATA { t.Fatalf("Removexattr failed: %v", err) } - - // Wait for background deletion and block freeing - time.Sleep(500 * time.Millisecond) + waitForSnapshotState(t, filePath, "", "temp-snap") // Verify the live file still has the new content readBack, err := mfs.ReadFile("reuse.txt") @@ -940,13 +938,7 @@ func TestSnapshotDeleteAfterOverwrite(t *testing.T) { if err := unix.Removexattr(mfs.Path("data.bin"), "user.snapshot.temp"); err != nil { t.Fatalf("delete snapshot: %v", err) } - listed := make([]byte, 4096) - if _, ok := waitFor(10*time.Second, func() bool { - n, err := unix.Getxattr(mfs.Path("data.bin"), "user.snapshots", listed) - return err == nil && !bytes.Contains(listed[:n], []byte("temp")) - }); !ok { - t.Fatal("snapshot still listed 10s after delete") - } + waitForSnapshotState(t, mfs.Path("data.bin"), "", "temp") if got := freeBlocks(t, mfs); got != free { t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index c72d15c9..7001751d 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1306,7 +1306,7 @@ scenarios: description: A delete job queued with an already canceled request context still runs priority: go_only environments: - go: internal/snapshot/worker_test.go::TestQueueDeleteJobIgnoresCanceledContext + go: internal/snapshot/worker_test.go::TestWorkerQueuesJobsWithCanceledContext - id: refcount-share-release name: Block reference sharing From b76afaa0fa78b2b2f7c47ce84d5d2def90838aab Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:14:59 -0500 Subject: [PATCH 07/18] fix(extent): conflict-check copy-on-write and never free uncounted sharing Two nodes could copy the same shared block of an inline file: neither inode write was checked, so both released the old block and freed it while a snapshot still mapped it. A copy-on-write now leaves a pending change, so the second write fails with a tree conflict and retries. New sharing sets FlagCounted. Blocks older versions marked shared without it have unknown owner counts, so frees leave them allocated and HasUncountedSharing lets callers refuse to share them again. Co-Authored-By: Claude Opus 5.5 --- internal/extent/extent.go | 62 ++++++++--------- internal/extent/extent_test.go | 123 ++++++++++++++++++++++++++++++--- internal/extent/tree_edit.go | 12 ++++ internal/refcount/refcount.go | 13 ---- 4 files changed, 156 insertions(+), 54 deletions(-) diff --git a/internal/extent/extent.go b/internal/extent/extent.go index a06d41b8..9790f681 100644 --- a/internal/extent/extent.go +++ b/internal/extent/extent.go @@ -46,6 +46,9 @@ const ( FlagUnwritten = 1 << 1 // Allocated but lazy-zero (not yet written) FlagShared = 1 << 2 // Shared with snapshot (requires CoW) FlagSFB = 1 << 3 // Stored in SFB container (not full block) + // FlagCounted marks sharing that refcounts track. Older versions set + // FlagShared alone and never counted the owners. + FlagCounted = 1 << 6 ) // Hole is what LookupPhysical returns for a logical block with no data on disk. @@ -333,49 +336,39 @@ func (m *Manager) release(ctx context.Context, ino *inode.Inode, removed []Exten ino.WrittenBlocks -= min(ino.WrittenBlocks, x.Length) } if x.Flags&FlagSFB == 0 { - blocks = append(blocks, m.ownedBlocks(ctx, x)...) + blocks = append(blocks, m.ownedBlocks(x)...) } } if len(blocks) == 0 { return } - p := pendingOf(ino) - if p == nil { - p = &pendingTree{base: committedTree(ino)} - ino.PendingTree = p - } + p := ensurePending(ino) p.release = append(p.release, blocks...) } -// ownedBlocks returns the blocks of x that refcounts can account for. A -// shared block with a count of 0 comes from a clone made by an older version, -// which shared blocks without counting the owners, so it stays allocated. So -// do all shared blocks when refcounts are disabled. -func (m *Manager) ownedBlocks(ctx context.Context, x Extent) []uint64 { +// ownedBlocks returns the blocks of x that frees may give back. Nobody knows +// how many inodes share an uncounted block, so it stays allocated. +func (m *Manager) ownedBlocks(x Extent) []uint64 { + if x.Flags&FlagShared != 0 && (x.Flags&FlagCounted == 0 || m.refs == nil) { + return nil + } blocks := make([]uint64, x.Length) for j := range blocks { blocks[j] = x.PhysicalBlock + uint64(j) } - if x.Flags&FlagShared == 0 { - return blocks - } - if m.refs == nil { - return nil - } - counts, err := m.refs.Counts(ctx, blocks) - if err != nil { - log.Printf("[extent] read refcounts of blocks %d-%d: %v; leaving them allocated", x.PhysicalBlock, x.PhysicalBlock+x.Length-1, err) - return nil - } - owned := blocks[:0] - for j, b := range blocks { - if counts[j] == 0 { - log.Printf("[extent] block %d is shared but uncounted; leaving it allocated", b) - continue + return blocks +} + +// HasUncountedSharing reports whether ino shares blocks the way older versions +// did, without refcounts. Such blocks cannot be shared again safely. +func (m *Manager) HasUncountedSharing(ino *inode.Inode) (bool, error) { + uncounted := false + err := m.forEach(ino, func(x Extent) { + if x.Flags&FlagShared != 0 && x.Flags&FlagCounted == 0 { + uncounted = true } - owned = append(owned, b) - } - return owned + }) + return uncounted, err } // Delete unmaps every block of an inode that is being freed and frees the @@ -599,11 +592,14 @@ func (m *Manager) CopyOnWrite(ctx context.Context, ino *inode.Inode, ext *Extent return nil, fmt.Errorf("CoW write failed: %w", err) } - private := Extent{logicalBlock, newPhysBlock, 1, cur.Flags &^ FlagShared} + private := Extent{logicalBlock, newPhysBlock, 1, cur.Flags &^ (FlagShared | FlagCounted)} if err := m.remap(ctx, ino, private); err != nil { m.bitmap.Free(ctx, newPhysBlock) return nil, err } + // The caller releases the old block after the inode write, so that write + // must fail if another node copied the block first. + ensurePending(ino) *ext = private // Readers keep cached block mappings until DataGeneration changes. ino.DataGeneration++ @@ -636,7 +632,7 @@ func (m *Manager) MarkAllShared(ino *inode.Inode) error { if err != nil { return err } - if err := e.updateAll(func(x *Extent) { x.Flags |= FlagShared }); err != nil { + if err := e.updateAll(func(x *Extent) { x.Flags |= FlagShared | FlagCounted }); err != nil { return err } return e.commit() @@ -652,7 +648,7 @@ func (m *Manager) CopyExtentsForClone(ctx context.Context, src *inode.Inode, dst } all := []Extent{} if err := m.forEach(src, func(x Extent) { - x.Flags |= FlagShared + x.Flags |= FlagShared | FlagCounted all = append(all, x) }); err != nil { return err diff --git a/internal/extent/extent_test.go b/internal/extent/extent_test.go index bfeac5bc..90f7bc3c 100644 --- a/internal/extent/extent_test.go +++ b/internal/extent/extent_test.go @@ -2,6 +2,7 @@ package extent import ( "context" + "errors" "os" "path/filepath" "testing" @@ -1551,17 +1552,123 @@ func TestFreeKeepsSharedBlocks(t *testing.T) { t.Fatalf("free blocks after truncating both = %d, want %d", got, initial) } - // Older versions shared blocks with clones without counting them. - legacy := env.newInode() - if _, err := env.mgr.AllocateBlocks(env.ctx, legacy, 0, 1); err != nil { +} + +// legacyShared maps a new block into each inode the way older versions shared +// blocks: FlagShared without FlagCounted, and a refcount of count. +func legacyShared(t *testing.T, env *treeEnv, refs *refcount.Region, count uint32, inodes ...*inode.Inode) uint64 { + t.Helper() + blocks, err := env.bm.Alloc(env.ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := refs.Set(env.ctx, blocks[0], count, refcount.FlagNone); err != nil { + t.Fatal(err) + } + for _, ino := range inodes { + env.insert(ino, Extent{LogicalBlock: 0, PhysicalBlock: blocks[0], Length: 1, Flags: FlagAllocated | FlagShared}) + env.write(ino) + } + return blocks[0] +} + +func TestUncountedSharingIsNeverFreed(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + truncate := func(ino *inode.Inode) { + t.Helper() + if err := env.mgr.Truncate(env.ctx, ino, 0); err != nil { + t.Fatal(err) + } + env.write(ino) + } + + // An old clone left a count of 0; an old snapshot and then an old clone left 1. + for _, count := range []uint32{0, 1} { + src, clone := env.newInode(), env.newInode() + block := legacyShared(t, env, refs, count, src, clone) + for _, ino := range []*inode.Inode{src, clone} { + uncounted, err := env.mgr.HasUncountedSharing(ino) + if err != nil || !uncounted { + t.Fatalf("count %d: HasUncountedSharing = %v, %v; want true", count, uncounted, err) + } + } + truncate(clone) + if ok, _ := env.bm.IsAllocated(block); !ok { + t.Fatalf("count %d: truncating the old clone freed block %d that its source maps", count, block) + } + } + + fresh := env.newInode() + if _, err := env.mgr.AllocateBlocks(env.ctx, fresh, 0, 1); err != nil { + t.Fatal(err) + } + if err := env.mgr.MarkAllShared(fresh); err != nil { t.Fatal(err) } - if err := env.mgr.MarkAllShared(legacy); err != nil { + if uncounted, err := env.mgr.HasUncountedSharing(fresh); err != nil || uncounted { + t.Fatalf("sharing marked by this version reported as uncounted: %v, %v", uncounted, err) + } +} + +// Two nodes copy the same shared block of an inline file: only the first +// inode write may land, so the old block is released once. +func TestConcurrentInlineCopyOnWrite(t *testing.T) { + env := newTreeEnv(t, 0) + refs := refcount.NewRegion(env.dev, treeTestInodeOffset+512<<10, treeTestBlocks, nil) + env.mgr.SetRefcounts(refs) + + src := env.newInode() + ext, err := env.mgr.AllocateBlocks(env.ctx, src, 0, 1) + if err != nil { + t.Fatal(err) + } + old := ext.PhysicalBlock + if err := env.mgr.MarkAllShared(src); err != nil { + t.Fatal(err) + } + env.write(src) + if err := refs.Share(env.ctx, []uint64{old}); err != nil { t.Fatal(err) } - env.write(legacy) - truncate(legacy) - if got := env.bm.FreeCount(); got != initial-1 { - t.Fatalf("free blocks after truncating an uncounted shared block = %d, want %d", got, initial-1) + snap := env.newInode() + if err := env.mgr.CopyExtentsForClone(env.ctx, src, snap); err != nil { + t.Fatal(err) + } + env.write(snap) + + nodeA, nodeB := env.reread(src), env.reread(src) + var copies []uint64 + for _, ino := range []*inode.Inode{nodeA, nodeB} { + x, _, err := env.mgr.Lookup(ino, 0) + if err != nil { + t.Fatal(err) + } + res, err := env.mgr.CopyOnWrite(env.ctx, ino, x, 0, treeTestDataOffset, refs) + if err != nil || !res.WasShared { + t.Fatalf("copy-on-write: %+v, %v", res, err) + } + copies = append(copies, res.NewPhysBlock) + } + + env.write(nodeA) + env.mgr.ReleaseBlocks(env.ctx, []uint64{old}) + if err := env.tbl.Write(env.ctx, nodeB); !errors.Is(err, ErrTreeConflict) { + t.Fatalf("second copy's inode write: got %v, want ErrTreeConflict", err) + } + env.mgr.DiscardPending(env.ctx, nodeB) + if err := env.bm.Free(env.ctx, copies[1]); err != nil { + t.Fatal(err) + } + + if ok, _ := env.bm.IsAllocated(old); !ok { + t.Fatalf("block %d freed while the snapshot still maps it", old) + } + if e, err := refs.Get(env.ctx, old); err != nil || e.RefCount != 1 { + t.Fatalf("old block refcount = %+v, %v; want 1 (the snapshot)", e, err) + } + if got, _, err := env.mgr.Lookup(env.reread(src), 0); err != nil || got.PhysicalBlock != copies[0] { + t.Fatalf("file maps %+v, %v; want the first copy %d", got, err, copies[0]) } } diff --git a/internal/extent/tree_edit.go b/internal/extent/tree_edit.go index 3072eefe..432cddd9 100644 --- a/internal/extent/tree_edit.go +++ b/internal/extent/tree_edit.go @@ -714,6 +714,18 @@ func (m *Manager) allocTreeBlocks(ctx context.Context, n int) ([]uint64, error) return out, nil } +// ensurePending returns ino's pending change, creating an empty one if needed. +// Any pending change makes the inode write check that the extents on disk are +// still the ones this change started from. +func ensurePending(ino *inode.Inode) *pendingTree { + p := pendingOf(ino) + if p == nil { + p = &pendingTree{base: committedTree(ino)} + ino.PendingTree = p + } + return p +} + // ReleaseBlocks drops one owner from each block and frees the blocks that no // owner holds any more, such as a block a write just copied away from. func (m *Manager) ReleaseBlocks(ctx context.Context, blocks []uint64) { diff --git a/internal/refcount/refcount.go b/internal/refcount/refcount.go index 4ada00f8..d8682f4b 100644 --- a/internal/refcount/refcount.go +++ b/internal/refcount/refcount.go @@ -343,19 +343,6 @@ func (r *Region) Release(ctx context.Context, blockNums []uint64) ([]uint64, err return free, nil } -// Counts returns the reference count of each block. -func (r *Region) Counts(ctx context.Context, blockNums []uint64) ([]uint32, error) { - counts := make([]uint32, len(blockNums)) - err := r.updateBatch(ctx, blockNums, func(i int, e *Entry) (bool, error) { - counts[i] = e.RefCount - return false, nil - }) - if err != nil { - return nil, err - } - return counts, nil -} - // updateBatch applies fn to the entry of each block under the region lock and // writes back the sectors fn changed. Nothing is written if fn fails. func (r *Region) updateBatch(ctx context.Context, blockNums []uint64, fn func(i int, e *Entry) (bool, error)) error { From 9d2ba62ead82febfd87e7c7de439f1d22b7db773 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:15:00 -0500 Subject: [PATCH 08/18] fix(snapshot): order sharing safely and refuse files shared without refcounts - Mark the source shared before its blocks gain owners, and sync the refcounts before the copy maps them, so a crash leaves at most a leak. - Refuse to snapshot or clone a file an older version shared (ENOTSUP). - Re-check that a snapshot is still active before a clone of it maps its blocks. - Only a snapshot still being created can become active. - Snapshot IDs come from the highest ID on disk, so two nodes no longer reuse one. - Refcounts stay on with -snapshots=false, so shared blocks are still copied before writes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- cmd/mount/main.go | 9 +- internal/goclusterfs/fs.go | 70 ++++++++-- internal/goclusterfs/snapshot_test.go | 185 ++++++++++++++++++++++++-- internal/snapshot/snapshot.go | 59 +++++--- internal/snapshot/snapshot_test.go | 57 ++++++++ tests/test-manifest.yaml | 42 ++++++ 7 files changed, 375 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7ee028..b5ae68aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. +- Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. This version never frees such blocks, so deleting those files leaks their space. It also refuses to snapshot or clone them (`ENOTSUP`), and in-place writes to an old clone still change its source. To get files this version can snapshot and free, copy each affected file with `cp` and delete the original. - Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. -- Clones made by older versions share blocks without counting them. Newer binaries keep such a block allocated when a file lets go of it, so deleting an old clone or its source leaks its space instead of freeing blocks the other file still uses. Copy the clone with `cp` and delete the old one to avoid the leak. ### Added diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e77fd5d9..85b6031f 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -266,12 +266,9 @@ func main() { log.Println("SFB: disabled (all files use full 1MB blocks)") } - // Create refcount region for snapshot CoW support - var refcountReg *refcount.Region - if *snapshotsEnabled { - refcountReg = refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) - log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) - } + // Refcounts stay on without snapshots: blocks shared earlier still need copy-on-write. + refcountReg := refcount.NewRegion(dev, refcountOffset, totalBlocks, lockMgr) + log.Printf("Refcount: region at %dMB (%d blocks)", refcountOffset/(1024*1024), totalBlocks) // Create filesystem with journal, TRIM, SFB, and refcount for crash consistency gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, sfbMgr, refcountReg, lockMgr, nodeUUID, dataOffset) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 519d3074..e30b0dd9 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3292,15 +3292,33 @@ func (f *FS) markShared(ino *inode.Inode) error { return nil } +// errUncountedSharing means the file shares blocks the way older versions did, +// without refcounts, so its blocks cannot be shared again safely. +var errUncountedSharing = errors.New("file shares blocks without refcounts") + // shareInto gives dst the extents of inode srcNum, sharing its data blocks. -// Each block gains an owner before dst maps it, and src is marked shared last, -// so from then on writes to either inode copy blocks before changing them. On -// error the caller discards dst, which gives back any blocks it maps. -func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode) error { +// src is marked shared first, then each block gains an owner, durably, before +// dst maps it; a crash in between leaves src copying blocks it alone owns. +// stillValid, if set, runs after the blocks gain owners and before dst maps +// them. On error the caller discards dst, which gives back any blocks it maps. +func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode, stillValid func() error) error { src, err := f.inodeTbl.Read(srcNum) if err != nil { return err } + uncounted, err := f.extentMgr.HasUncountedSharing(src) + if err != nil { + return err + } + if uncounted { + return errUncountedSharing + } + // A snapshot's data inode is marked shared when it is created and never written again. + if src.Flags&inode.FlagSnapshotData == 0 { + if src, err = f.updateInode(ctx, srcNum, f.markShared); err != nil { + return err + } + } blocks, err := f.extentMgr.DataBlocks(src) if err != nil { return err @@ -3308,6 +3326,14 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode) err if err := f.refcountReg.Share(ctx, blocks); err != nil { return err } + err = f.dev.Sync() + if err == nil && stillValid != nil { + err = stillValid() + } + if err != nil { + f.extentMgr.ReleaseBlocks(ctx, blocks) + return err + } if err := f.extentMgr.CopyExtentsForClone(ctx, src, dst); err != nil { f.extentMgr.ReleaseBlocks(ctx, blocks) return err @@ -3321,8 +3347,15 @@ func (f *FS) shareInto(ctx context.Context, srcNum uint32, dst *inode.Inode) err } return err } - _, err = f.updateInode(ctx, srcNum, f.markShared) - return err + return nil +} + +// sharingErrno maps a failure to share a file's blocks to an errno. +func sharingErrno(err error) syscall.Errno { + if errors.Is(err, errUncountedSharing) { + return syscall.ENOTSUP + } + return syscall.EIO } // claimInode allocates an inode slot, retrying when another node takes the @@ -3423,16 +3456,16 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EIO } - err = f.fs.shareInto(ctx, f.inodeNum, data) + err = f.fs.shareInto(ctx, f.inodeNum, data, nil) if err == nil { - err = f.fs.snapshotReg.UpdateState(ctx, desc.SnapshotID, snapshot.StateActive) + err = f.fs.snapshotReg.Activate(ctx, desc.SnapshotID) } if err != nil { log.Printf("[snapshot] Failed to create '%s' for inode %d: %v", name, f.inodeNum, err) if dropErr := f.fs.dropSnapshot(ctx, desc.SnapshotID); dropErr != nil { log.Printf("[snapshot] Failed to roll back snapshot %d: %v", desc.SnapshotID, dropErr) } - return syscall.EIO + return sharingErrno(err) } log.Printf("[snapshot] Created '%s' (ID=%d) for inode %d", name, desc.SnapshotID, f.inodeNum) @@ -3475,11 +3508,12 @@ func (f *File) createClone(ctx context.Context, destPath string) syscall.Errno { log.Printf("[clone] Failed to read inode %d: %v", f.inodeNum, err) return syscall.EIO } - return f.cloneInode(ctx, f.inodeNum, destPath) + return f.cloneInode(ctx, f.inodeNum, destPath, nil) } // cloneInode creates a file at destPath that shares the blocks of inode srcNum. -func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string) syscall.Errno { +// stillValid is passed to shareInto. +func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string, stillValid func() error) syscall.Errno { if f.fs.refcountReg == nil { return syscall.ENOTSUP } @@ -3551,11 +3585,11 @@ func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string) s cloneIno.Mtime = now cloneIno.Ctime = now - if err := f.fs.shareInto(ctx, srcNum, cloneIno); err != nil { + if err := f.fs.shareInto(ctx, srcNum, cloneIno, stillValid); err != nil { f.fs.dirMgr.Remove(ctx, rootIno, cloneName) f.fs.discardInode(ctx, cloneInodeNum) log.Printf("[clone] Failed to share blocks of inode %d: %v", srcNum, err) - return syscall.EIO + return sharingErrno(err) } log.Printf("[clone] Created '%s' (inode %d) from inode %d", cloneName, cloneInodeNum, srcNum) @@ -3598,7 +3632,15 @@ func (f *File) createCloneFromSnapshot(ctx context.Context, data string) syscall return syscall.ENOTSUP } - return f.cloneInode(ctx, desc.DataInode, destPath) + // A delete that starts before the blocks gain their new owner may free them. + stillActive := func() error { + d, err := f.fs.snapshotReg.Get(ctx, desc.SnapshotID) + if err == nil && d.State != snapshot.StateActive { + err = snapshot.ErrSnapshotInvalidState + } + return err + } + return f.cloneInode(ctx, desc.DataInode, destPath, stillActive) } // listSnapshots returns snapshot names for this file. diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 04c56661..413fde5b 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "syscall" "testing" "time" @@ -157,6 +158,27 @@ func TestSnapshotKeepsDataThroughWrites(t *testing.T) { } }) + // Two blocks keep the copied block's split extents inline, so no tree node is allocated. + t.Run("DeleteAfterPartialOverwrite", func(t *testing.T) { + f := newRegularFile(t, fsys) + writeAt(t, f, original[:2*BlockSize], 0) + free := fsys.bitmapMgr.FreeCount() + setxattr(t, f, "user.snapshot", "partial") + + writeAt(t, f, pattern(7, 4096), BlockSize+10) + if got := fsys.bitmapMgr.FreeCount(); got != free-1 { + t.Fatalf("free blocks after a write into 1 shared block = %d, want %d", got, free-1) + } + if errno := f.Removexattr(ctx, "user.snapshot.partial"); errno != 0 { + t.Fatalf("delete snapshot: %v", errno) + } + waitSnapshotGone(t, fsys, "partial") + if got := fsys.bitmapMgr.FreeCount(); got != free { + t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) + } + checkFile(t, f, overwritten(original[:2*BlockSize], BlockSize+10, pattern(7, 4096))) + }) + t.Run("DeleteAfterOverwriteFreesSnapshotBlocks", func(t *testing.T) { f := newRegularFile(t, fsys) writeAt(t, f, original[:2*BlockSize], 0) @@ -171,20 +193,161 @@ func TestSnapshotKeepsDataThroughWrites(t *testing.T) { if errno := f.Removexattr(ctx, "user.snapshot.temp"); errno != 0 { t.Fatalf("delete snapshot: %v", errno) } - deadline := time.Now().Add(10 * time.Second) - for { - _, err := fsys.snapshotReg.GetByName(ctx, "temp") - if errors.Is(err, snapshot.ErrSnapshotNotFound) { - break - } - if time.Now().After(deadline) { - t.Fatalf("snapshot still listed 10s after delete: %v", err) - } - time.Sleep(10 * time.Millisecond) - } + waitSnapshotGone(t, fsys, "temp") if got := fsys.bitmapMgr.FreeCount(); got != free { t.Fatalf("free blocks after deleting the snapshot = %d, want %d", got, free) } checkFile(t, f, pattern(6, 2*BlockSize)) }) } + +func waitSnapshotGone(t *testing.T, fsys *FS, name string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + _, err := fsys.snapshotReg.GetByName(context.Background(), name) + if errors.Is(err, snapshot.ErrSnapshotNotFound) { + return + } + if time.Now().After(deadline) { + t.Fatalf("snapshot %q still listed after 10s: %v", name, err) + } + time.Sleep(10 * time.Millisecond) + } +} + +func physicalBlock(t *testing.T, fsys *FS, num uint32, logical uint64) uint64 { + t.Helper() + ino, err := fsys.inodeTbl.Read(num) + if err != nil { + t.Fatal(err) + } + ext, off, err := fsys.extentMgr.Lookup(ino, logical) + if err != nil { + t.Fatal(err) + } + return ext.PhysicalBlock + off +} + +func refcountOf(t *testing.T, fsys *FS, block uint64) uint32 { + t.Helper() + e, err := fsys.refcountReg.Get(context.Background(), block) + if err != nil { + t.Fatal(err) + } + return e.RefCount +} + +// Older versions shared blocks without counting the owners, so this version +// refuses to share them again. +func TestSnapshotRefusesUncountedSharing(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + f := newRegularFile(t, fsys) + writeAt(t, f, pattern(1, BlockSize), 0) + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + ino.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + usedInodes := func() int { + n := 0 + for num := uint32(inode.InodeFirstUsable); num < 64; num++ { + if ino, err := fsys.inodeTbl.Read(num); err == nil && ino.Magic != 0 { + n++ + } + } + return n + } + used := usedInodes() + + if errno := f.Setxattr(ctx, "user.snapshot", []byte("legacy"), 0); errno != syscall.ENOTSUP { + t.Fatalf("snapshot of a file with uncounted sharing: errno %v, want ENOTSUP", errno) + } + if errno := f.Setxattr(ctx, "user.clone", []byte("/legacy-clone"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of a file with uncounted sharing: errno %v, want ENOTSUP", errno) + } + waitSnapshotGone(t, fsys, "legacy") + // The delete job frees the data inode right after removing the descriptor. + deadline := time.Now().Add(10 * time.Second) + for usedInodes() != used { + if time.Now().After(deadline) { + t.Fatalf("refused snapshot and clone left %d inodes in use, want %d", usedInodes(), used) + } + time.Sleep(10 * time.Millisecond) + } + checkFile(t, f, pattern(1, BlockSize)) +} + +// A new mount cleans up a snapshot a crash left half created or half deleted, +// and the file keeps its blocks. +func TestSnapshotRecoveryAfterCrash(t *testing.T) { + path := newSnapshotFSImage(t) + fsys := openSnapshotFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + content := pattern(3, 2*BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, content, 0) + blocks := []uint64{physicalBlock(t, fsys, f.inodeNum, 0), physicalBlock(t, fsys, f.inodeNum, 1)} + + // Crash after the blocks were shared, before the snapshot became active. + data, err := fsys.claimInode(ctx) + if err != nil { + t.Fatal(err) + } + data.Mode = inode.S_IFREG | 0o644 + data.Flags |= inode.FlagSnapshotData + if err := fsys.inodeTbl.Write(ctx, data); err != nil { + t.Fatal(err) + } + if _, err := fsys.snapshotReg.Create(ctx, "half-created", f.inodeNum, 0, data.InodeNum, 2, uint64(len(content)), snapshot.FlagReadOnly); err != nil { + t.Fatal(err) + } + if err := fsys.shareInto(ctx, f.inodeNum, data, nil); err != nil { + t.Fatal(err) + } + + // Crash after the data inode gave back its blocks, before the descriptor went away. + setxattr(t, f, "user.snapshot", "half-deleted") + deleting, err := fsys.snapshotReg.GetByName(ctx, "half-deleted") + if err != nil { + t.Fatal(err) + } + if err := fsys.snapshotReg.Delete(ctx, deleting.SnapshotID); err != nil { + t.Fatal(err) + } + if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.DataInode); err != nil { + t.Fatal(err) + } + + remounted := openSnapshotFS(t, path) + waitSnapshotGone(t, remounted, "half-created") + waitSnapshotGone(t, remounted, "half-deleted") + for _, num := range []uint32{data.InodeNum, deleting.DataInode} { + if ino, err := remounted.inodeTbl.Read(num); err != nil || ino.Magic != 0 { + t.Errorf("data inode %d still in use after recovery (err=%v)", num, err) + } + } + for _, b := range blocks { + if got := refcountOf(t, remounted, b); got > 1 { + t.Errorf("block %d refcount %d after recovery, want at most 1", b, got) + } + } + + // With no other owner left, a write goes in place. + g := &File{fs: remounted, inodeNum: f.inodeNum} + writeAt(t, g, pattern(4, 100), 10) + if got := physicalBlock(t, remounted, f.inodeNum, 0); got != blocks[0] { + t.Errorf("write after recovery moved block 0 from %d to %d", blocks[0], got) + } + checkFile(t, g, overwritten(content, 10, pattern(4, 100))) +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index af72ac71..425672ea 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -285,15 +285,12 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, r.mu.Lock() defer r.mu.Unlock() - // Find a free slot - slot, err := r.findFreeSlot() + slot, maxID, err := r.freeSlotAndMaxID() if err != nil { return nil, err } - - // Allocate a new ID - snapshotID := r.nextID - r.nextID++ + snapshotID := max(r.nextID, maxID+1) + r.nextID = snapshotID + 1 if dataInode != 0 { flags |= FlagHasData @@ -461,6 +458,19 @@ func (r *Registry) UpdateParentID(ctx context.Context, snapshotID, newParentID u // UpdateState updates the state of a snapshot. func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState State) error { + return r.updateState(ctx, snapshotID, newState, StateInvalid) +} + +// Activate marks a snapshot that is being created as active. It fails with +// ErrSnapshotInvalidState if the snapshot left the creating state meanwhile, +// for example because another node's mount rolled it back. +func (r *Registry) Activate(ctx context.Context, snapshotID uint64) error { + return r.updateState(ctx, snapshotID, StateActive, StateCreating) +} + +// updateState sets a snapshot's state, requiring it to be in from unless from +// is StateInvalid. +func (r *Registry) updateState(ctx context.Context, snapshotID uint64, newState, from State) error { if snapshotID == InvalidSnapshotID { return ErrInvalidID } @@ -483,6 +493,9 @@ func (r *Registry) UpdateState(ctx context.Context, snapshotID uint64, newState continue } if desc.IsValid() && desc.SnapshotID == snapshotID { + if from != StateInvalid && desc.State != from { + return ErrSnapshotInvalidState + } desc.State = newState if newState == StateDeleting { desc.DeleteTime = time.Now().Unix() @@ -661,19 +674,31 @@ type ScanStats struct { TotalBlocks uint64 // Total blocks referenced by all snapshots } -// findFreeSlot finds the first free slot in the registry. -func (r *Registry) findFreeSlot() (uint64, error) { - for slot := uint64(0); slot < r.maxDescriptors; slot++ { - desc, err := r.readDescriptor(slot) - if err != nil { - // Treat read errors as potentially free (might be uninitialized) - return slot, nil - } - if desc.IsFree() { - return slot, nil +// freeSlotAndMaxID returns the first free slot and the highest snapshot ID on +// disk, which includes snapshots other nodes created since this node's scan. +// Unreadable slots count as free, as they did before. +func (r *Registry) freeSlotAndMaxID() (uint64, uint64, error) { + free := r.maxDescriptors + var maxID uint64 + for sector := uint64(0); sector*DescriptorsPerSector < r.maxDescriptors; sector++ { + data, readErr := r.readSector(sector) + for i := uint64(0); i < DescriptorsPerSector; i++ { + slot := sector*DescriptorsPerSector + i + if slot >= r.maxDescriptors { + break + } + desc, err := decodeDescriptor(data, i) + if readErr != nil || err != nil || desc.IsFree() { + free = min(free, slot) + continue + } + maxID = max(maxID, desc.SnapshotID) } } - return 0, ErrSnapshotFull + if free == r.maxDescriptors { + return 0, 0, ErrSnapshotFull + } + return free, maxID, nil } // readDescriptor reads a descriptor from a slot. diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 7d3180f1..aad27f46 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -2,6 +2,7 @@ package snapshot import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -850,3 +851,59 @@ func BenchmarkList(b *testing.B) { registry.List(ctx) } } + +// Two mounted nodes each create a snapshot; the IDs must differ because a +// delete by ID would otherwise remove the other node's snapshot. +func TestRegistryIDsUniqueAcrossNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + ctx := context.Background() + + nodeA := NewRegistry(dev, 0, 10*SectorSize, nil) + nodeB := NewRegistry(dev, 0, 10*SectorSize, nil) + for _, r := range []*Registry{nodeA, nodeB} { + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + } + a, err := nodeA.Create(ctx, "from-a", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + b, err := nodeB.Create(ctx, "from-b", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if a.SnapshotID == b.SnapshotID { + t.Fatalf("both nodes created snapshot ID %d", a.SnapshotID) + } +} + +func TestRegistryActivateRequiresCreating(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + registry := NewRegistry(dev, 0, 10*SectorSize, nil) + ctx := context.Background() + + desc, err := registry.Create(ctx, "rolled-back", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := registry.Delete(ctx, desc.SnapshotID); err != nil { + t.Fatal(err) + } + if err := registry.Activate(ctx, desc.SnapshotID); !errors.Is(err, ErrSnapshotInvalidState) { + t.Fatalf("Activate of a deleting snapshot: got %v, want ErrSnapshotInvalidState", err) + } + if got, err := registry.Get(ctx, desc.SnapshotID); err != nil || got.State != StateDeleting { + t.Fatalf("snapshot after refused Activate: %+v, %v; want deleting", got, err) + } + + fresh, err := registry.Create(ctx, "fresh", 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := registry.Activate(ctx, fresh.SnapshotID); err != nil { + t.Fatalf("Activate of a creating snapshot: %v", err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 7001751d..1c1a6836 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1322,6 +1322,48 @@ scenarios: environments: go: internal/extent/extent_test.go::TestFreeKeepsSharedBlocks + - id: snapshot-refuses-uncounted-sharing + name: Snapshot of a file shared by an older version + description: Snapshotting or cloning a file whose blocks an older version shared without refcounts fails with ENOTSUP and leaves nothing behind + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRefusesUncountedSharing + + - id: extent-uncounted-sharing-never-freed + name: Blocks shared by an older version stay allocated + description: Truncating a file that shares blocks without refcounts, with a count of 0 or 1, never frees them + priority: go_only + environments: + go: internal/extent/extent_test.go::TestUncountedSharingIsNeverFreed + + - id: extent-concurrent-inline-cow + name: Two nodes copy the same shared block + description: When two copies of an inline inode copy the same shared block, the second inode write fails with a tree conflict and the old block is released once + priority: go_only + environments: + go: internal/extent/extent_test.go::TestConcurrentInlineCopyOnWrite + + - id: snapshot-recovery-after-crash + name: Snapshot cleanup on mount after a crash + description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestSnapshotRecoveryAfterCrash + + - id: snapshot-ids-unique-across-nodes + name: Snapshot IDs across nodes + description: Two mounted nodes creating snapshots get different IDs + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryIDsUniqueAcrossNodes + + - id: snapshot-activate-requires-creating + name: Snapshot activation after a rollback + description: A snapshot another node rolled back cannot be marked active + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryActivateRequiresCreating + - id: snapshot-cow-in-process name: Snapshot and clone copy-on-write description: Snapshots keep their data through in-place writes, appends and truncates; clone writes never reach the source; deleting a snapshot after an overwrite frees exactly its blocks From fe98dbd70fb6ca6fbc5812964ab236fc64bd28df Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:20:53 -0500 Subject: [PATCH 09/18] fix(snapshot): lock the snapshot registry across nodes The registry got FS.lockMgr, which is nil, so two nodes creating snapshots at once could take the same slot or ID: one snapshot vanished or a delete by ID removed the other node's. InitSnapshots now takes the mount's lock manager. Adds a two-node registry test and Go, Docker and VM tests that snapshot on two nodes at once. Adds tests where node A commits a copy of a shared block between node B's copy and commit, and where an older version's snapshot and clone of one file are written and deleted. The ENOTSUP for files an older version shared now points to goclusterfs-fsck, and the CHANGELOG says to run fsck before the first mount after upgrading. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + cmd/mount/main.go | 2 +- internal/goclusterfs/fs.go | 15 ++- internal/goclusterfs/snapshot_test.go | 141 +++++++++++++++++++++++++- internal/snapshot/snapshot_test.go | 57 +++++++++++ tests/docker/scripts/run-tests.sh | 59 +++++++++++ tests/integration/snapshot_test.go | 66 ++++++++++++ tests/test-manifest.yaml | 30 ++++++ tests/vm/scripts/run-vm-tests.sh | 54 ++++++++++ 9 files changed, 420 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ae68aa..225d4e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. +- Run `goclusterfs-fsck` before the first mount after upgrading an image that has snapshots or clones. Phase 5 reports blocks shared by more files than their refcount allows. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. This version never frees such blocks, so deleting those files leaks their space. It also refuses to snapshot or clone them (`ENOTSUP`), and in-place writes to an old clone still change its source. To get files this version can snapshot and free, copy each affected file with `cp` and delete the original. - Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 85b6031f..405d4ec3 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -304,7 +304,7 @@ func main() { // Initialize snapshot subsystem after verifying filesystem is valid if *snapshotsEnabled { - if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize); err != nil { + if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize, lockMgr); err != nil { log.Fatalf("Failed to initialize snapshots: %v", err) } log.Printf("Snapshots: enabled (registry at %dMB, %dMB capacity)", diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index e30b0dd9..f9c3fc59 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -42,6 +42,10 @@ import ( // modification was detected and the Write operation should be retried. var errRetryWrite = errors.New("retry write operation") +// beforeCoWCommit lets tests run another node's write between a copy-on-write +// and the inode write that commits it. +var beforeCoWCommit func() + // MaxWriteRetries is the maximum number of retry attempts for Write operations // when concurrent modifications are detected. const MaxWriteRetries = 3 @@ -229,15 +233,17 @@ func NewWithComponents( // Parameters: // - snapshotOffset: byte offset where snapshot region starts on device // - snapshotSize: size of snapshot region in bytes +// - lockMgr: serializes registry changes across nodes, so two nodes never +// take the same descriptor slot or snapshot ID (nil for single-node use) // // This must be called before any snapshot operations can succeed. -func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64) error { +func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64, lockMgr *lock.Manager) error { if f.refcountReg == nil { return fmt.Errorf("refcount region must be initialized before snapshots") } // Create snapshot registry - f.snapshotReg = snapshot.NewRegistry(f.dev, snapshotOffset, snapshotSize, f.lockMgr) + f.snapshotReg = snapshot.NewRegistry(f.dev, snapshotOffset, snapshotSize, lockMgr) // Scan registry to rebuild state stats, err := f.snapshotReg.Scan(ctx) @@ -2689,6 +2695,9 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return 0, writeFailed(err) } } + if len(cowDone) > 0 && beforeCoWCommit != nil { + beforeCoWCommit() + } // Write inode - use WriteWithLockHeld if we hold the lock if lockHeld { @@ -3294,7 +3303,7 @@ func (f *FS) markShared(ino *inode.Inode) error { // errUncountedSharing means the file shares blocks the way older versions did, // without refcounts, so its blocks cannot be shared again safely. -var errUncountedSharing = errors.New("file shares blocks without refcounts") +var errUncountedSharing = errors.New("an older version shared this file's blocks without refcounts; run goclusterfs-fsck, and copy the file to snapshot or clone it") // shareInto gives dst the extents of inode srcNum, sharing its data blocks. // src is marked shared first, then each block gains an owner, durably, before diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 413fde5b..55684bf0 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -61,7 +61,7 @@ func openSnapshotFS(t *testing.T, path string) *FS { refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, nil) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, nil, snapFSDataOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, nodeUUID, snapFSDataOffset) - if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize); err != nil { + if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, nil); err != nil { t.Fatalf("InitSnapshots: %v", err) } t.Cleanup(fsys.snapshotWkr.Stop) @@ -351,3 +351,142 @@ func TestSnapshotRecoveryAfterCrash(t *testing.T) { } checkFile(t, g, overwritten(content, 10, pattern(4, 100))) } + +// Two nodes write into the same shared block at once. Node A commits its copy +// first; node B's commit then fails, B frees only its own copy and retries, +// and the old block loses exactly one owner. +func TestConcurrentCopyOnWriteAcrossNodes(t *testing.T) { + path := newSnapshotFSImage(t) + nodeA := openSnapshotFS(t, path) + if err := nodeA.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + original := pattern(1, BlockSize) + f := newRegularFile(t, nodeA) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "race") + old := physicalBlock(t, nodeA, f.inodeNum, 0) + + nodeB := openSnapshotFS(t, path) + t.Cleanup(func() { beforeCoWCommit = nil }) + beforeCoWCommit = func() { + beforeCoWCommit = nil + writeAt(t, f, pattern(2, 100), 0) + } + writeAt(t, &File{fs: nodeB, inodeNum: f.inodeNum}, pattern(3, 100), 200) + if beforeCoWCommit != nil { + t.Fatal("node B's write never reached a copy-on-write commit") + } + + after := openSnapshotFS(t, path) + if got := refcountOf(t, after, old); got != 1 { + t.Fatalf("old block refcount %d, want 1: the snapshot is its only owner", got) + } + current := physicalBlock(t, after, f.inodeNum, 0) + allocated := 0 + for b := uint64(0); b < snapFSDataBlocks; b++ { + if ok, err := after.bitmapMgr.IsAllocated(b); err != nil { + t.Fatal(err) + } else if ok { + allocated++ + } + } + // The root directory, the snapshot's old block and the file's copy; node B's copy is gone. + if allocated != 3 || current == old { + t.Fatalf("%d blocks allocated with the file on block %d (old %d); want 3 and a copy", allocated, current, old) + } + want := overwritten(overwritten(original, 0, pattern(2, 100)), 200, pattern(3, 100)) + checkFile(t, &File{fs: after, inodeNum: f.inodeNum}, want) + setxattr(t, &File{fs: after, inodeNum: f.inodeNum}, "user.snapshot.clone", "race:/race-clone") + checkFile(t, lookupFile(t, after, "race-clone"), original) +} + +// An image written by an older version, with an old snapshot and an old clone +// of one file. Writes, then deleting the clone and the source, never free a +// block another inode still maps; the blocks leak instead. +func TestOldSnapshotAndCloneNeverFreeSharedBlocks(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + src := newRegularFile(t, fsys) + writeAt(t, src, pattern(5, 2*BlockSize), 0) + + // What older versions left: blocks marked shared without FlagCounted, a + // refcount of 1 from the snapshot job, a clone of the same blocks whose + // refcount job never ran, and a snapshot descriptor without data. + srcIno, err := fsys.inodeTbl.Read(src.inodeNum) + if err != nil { + t.Fatal(err) + } + for i := range srcIno.InlineExtents { + if srcIno.InlineExtents[i].Length > 0 { + srcIno.InlineExtents[i].Flags |= extent.FlagShared + } + } + if err := fsys.inodeTbl.Write(ctx, srcIno); err != nil { + t.Fatal(err) + } + blocks, err := fsys.extentMgr.DataBlocks(srcIno) + if err != nil { + t.Fatal(err) + } + if _, err := fsys.refcountReg.IncrementBatch(ctx, blocks); err != nil { + t.Fatal(err) + } + clone := newRegularFile(t, fsys) + cloneIno, err := fsys.inodeTbl.Read(clone.inodeNum) + if err != nil { + t.Fatal(err) + } + cloneIno.InlineExtents = srcIno.InlineExtents + cloneIno.ExtentCount = srcIno.ExtentCount + cloneIno.Size, cloneIno.AllocatedBlocks, cloneIno.WrittenBlocks = srcIno.Size, srcIno.AllocatedBlocks, srcIno.WrittenBlocks + if err := fsys.inodeTbl.Write(ctx, cloneIno); err != nil { + t.Fatal(err) + } + old, err := fsys.snapshotReg.Create(ctx, "old-snap", src.inodeNum, 0, 0, 2, srcIno.Size, snapshot.FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := fsys.snapshotReg.Activate(ctx, old.SnapshotID); err != nil { + t.Fatal(err) + } + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + for name, num := range map[string]uint32{"src.bin": src.inodeNum, "clone.bin": clone.inodeNum} { + if err := fsys.dirMgr.Add(ctx, root, name, uint64(num), dir.DT_REG); err != nil { + t.Fatal(err) + } + } + stillAllocated := func(step string) { + t.Helper() + for _, b := range blocks { + if ok, err := fsys.bitmapMgr.IsAllocated(b); err != nil || !ok { + t.Fatalf("after %s, shared block %d is free (err=%v)", step, b, err) + } + } + } + + writeAt(t, clone, pattern(6, 100), 10) + stillAllocated("writing the old clone") + if errno := src.Setxattr(ctx, "user.snapshot.clone", []byte("old-snap:/x"), 0); errno != syscall.ENOTSUP { + t.Fatalf("clone of a snapshot without data: errno %v, want ENOTSUP", errno) + } + if errno := src.Removexattr(ctx, "user.snapshot.old-snap"); errno != 0 { + t.Fatalf("delete old snapshot: %v", errno) + } + waitSnapshotGone(t, fsys, "old-snap") + stillAllocated("deleting the old snapshot") + if errno := fsys.Unlink(ctx, "clone.bin"); errno != 0 { + t.Fatalf("unlink old clone: %v", errno) + } + stillAllocated("deleting the old clone") + if errno := fsys.Unlink(ctx, "src.bin"); errno != 0 { + t.Fatalf("unlink source: %v", errno) + } + stillAllocated("deleting the source") +} diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index aad27f46..5ed8d904 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -6,8 +6,10 @@ import ( "fmt" "os" "path/filepath" + "sync" "testing" + "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -907,3 +909,58 @@ func TestRegistryActivateRequiresCreating(t *testing.T) { t.Fatalf("Activate of a creating snapshot: %v", err) } } + +// Two nodes, each with its own lock manager on the shared lock region, create +// snapshots at the same time. Every snapshot must keep its own slot and ID. +func TestRegistryConcurrentCreatesAcrossNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 4<<20) + defer cleanup() + ctx := context.Background() + const perNode = 20 + + var nodes []*Registry + for slot := range 2 { + var nodeUUID [16]byte + nodeUUID[0] = byte(slot + 1) + locks := lock.NewManager(dev, 1<<20, 1<<20, nodeUUID, slot) + r := NewRegistry(dev, 0, 10*SectorSize, locks) + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + nodes = append(nodes, r) + } + + var wg sync.WaitGroup + errs := make(chan error, 2*perNode) + for n, r := range nodes { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perNode { + if _, err := r.Create(ctx, fmt.Sprintf("node%d-%d", n, i), 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly); err != nil { + errs <- err + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + all, err := nodes[0].List(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 2*perNode { + t.Fatalf("%d snapshots on disk, want %d", len(all), 2*perNode) + } + names := make(map[uint64]string) + for _, d := range all { + if other, dup := names[d.SnapshotID]; dup { + t.Fatalf("snapshots %q and %q share ID %d", other, d.GetName(), d.SnapshotID) + } + names[d.SnapshotID] = d.GetName() + } +} diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 21e44aec..ba4fda7b 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2931,6 +2931,64 @@ test_snapshot_delete_after_overwrite() { return 0 } +# Two nodes each take 10 snapshots of their own file at the same time. Each +# file must list all of its snapshots, and deleting one node's snapshots must +# leave the other node's alone. +test_snapshot_concurrent_creates() { + log "Testing concurrent snapshot creation on two nodes..." + + local node_a="${NODE_ARRAY[0]}" + local node_b="${NODE_ARRAY[1]:-}" + if [ -z "$node_b" ]; then + log "SKIP: needs two nodes" + return 0 + fi + local run=$$ + local file_a=/mnt/goclusterfs/conc_a.bin + local file_b=/mnt/goclusterfs/conc_b.bin + + exec_on_node "$node_a" sh -c "rm -f $file_a && head -c 1048576 /dev/urandom > $file_a" || return 1 + exec_on_node "$node_b" sh -c "rm -f $file_b && head -c 1048576 /dev/urandom > $file_b" || return 1 + + exec_on_node "$node_a" sh -c "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_a_${run}_\$i $file_a || exit 1; done" & + local pid_a=$! + exec_on_node "$node_b" sh -c "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_b_${run}_\$i $file_b || exit 1; done" & + local pid_b=$! + wait "$pid_a" || { log "ERROR: snapshots on $node_a failed"; return 1; } + wait "$pid_b" || { log "ERROR: snapshots on $node_b failed"; return 1; } + + local count_a count_b + count_a=$(exec_on_node "$node_a" sh -c "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -c ':active'") + count_b=$(exec_on_node "$node_b" sh -c "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_a" -ne 10 ] || [ "$count_b" -ne 10 ]; then + log "ERROR: $count_a and $count_b active snapshots, want 10 each" + return 1 + fi + + exec_on_node "$node_a" sh -c "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_a_${run}_\$i $file_a || exit 1; done" || { + log "ERROR: deleting snapshots on $node_a failed" + return 1 + } + local tries=0 + while exec_on_node "$node_a" sh -c "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -q conc_a_"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log "ERROR: $node_a snapshots still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + count_b=$(exec_on_node "$node_b" sh -c "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_b" -ne 10 ]; then + log "ERROR: $node_b lists $count_b snapshots after $node_a deleted its own, want 10" + return 1 + fi + + exec_on_node "$node_b" sh -c "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_b_${run}_\$i $file_b; done" + log "Concurrent snapshot creation test passed" + return 0 +} + run_test "Create and Read File" test_create_read_file run_test "Create Directory" test_create_directory run_test "File in Directory" test_file_in_directory @@ -2951,6 +3009,7 @@ run_test "Extent Tree Interleaved Appends" test_extent_tree_interleaved_appends run_test "Snapshot Preserves Data" test_snapshot_preserves_data run_test "Clone In-Place Write" test_clone_in_place_write run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite +run_test "Snapshot Concurrent Creates" test_snapshot_concurrent_creates if [ "$SKIP_CONCURRENT_TESTS" = "true" ]; then log "SKIPPING: Concurrent Writes (known race conditions - set SKIP_CONCURRENT_TESTS=false to enable)" diff --git a/tests/integration/snapshot_test.go b/tests/integration/snapshot_test.go index 21530bd5..43817567 100644 --- a/tests/integration/snapshot_test.go +++ b/tests/integration/snapshot_test.go @@ -3,8 +3,10 @@ package integration import ( "bytes" + "fmt" "os" "strings" + "sync" "syscall" "testing" "time" @@ -944,3 +946,67 @@ func TestSnapshotDeleteAfterOverwrite(t *testing.T) { } checkFile(t, mfs, "data.bin", updated) } + +// TestSnapshotConcurrentCreatesAcrossNodes has two nodes snapshot their own +// files at the same time. Each file keeps all its snapshots, and deleting one +// node's snapshots leaves the other node's alone. +func TestSnapshotConcurrentCreatesAcrossNodes(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and start first node: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to start second node: %v", err) + } + nodes := []*MountedFS{cluster.Node(0), cluster.Node(1)} + const perNode = 10 + file := func(n int) string { return fmt.Sprintf("conc%d.bin", n) } + snapName := func(n, i int) string { return fmt.Sprintf("conc%d-%d", n, i) } + for n, node := range nodes { + if err := node.WriteFile(file(n), chunk(byte(n+1), mb)); err != nil { + t.Fatal(err) + } + } + + var wg sync.WaitGroup + errs := make(chan error, 2*perNode) + for n, node := range nodes { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perNode { + if err := unix.Setxattr(node.Path(file(n)), "user.snapshot", []byte(snapName(n, i)), 0); err != nil { + errs <- fmt.Errorf("node %d snapshot %d: %w", n, i, err) + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + + names := func(n int) []string { + var out []string + for i := range perNode { + out = append(out, snapName(n, i)) + } + return out + } + for n, node := range nodes { + waitForSnapshotState(t, node.Path(file(n)), "active", names(n)...) + } + + for _, name := range names(0) { + if err := unix.Removexattr(nodes[0].Path(file(0)), "user.snapshot."+name); err != nil { + t.Fatalf("delete %s: %v", name, err) + } + } + waitForSnapshotState(t, nodes[0].Path(file(0)), "", names(0)...) + if states := snapshotStates(t, nodes[1].Path(file(1))); len(states) != perNode { + t.Fatalf("node 1's file lists %d snapshots after node 0 deleted its own, want %d: %v", len(states), perNode, states) + } + setxattr(t, nodes[1].Path(file(1)), "user.snapshot.clone", snapName(1, 0)+":/conc1-clone.bin") + checkFile(t, nodes[1], "conc1-clone.bin", chunk(2, mb)) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 1c1a6836..c83abdfb 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1343,6 +1343,20 @@ scenarios: environments: go: internal/extent/extent_test.go::TestConcurrentInlineCopyOnWrite + - id: snapshot-concurrent-cow-across-nodes + name: Two nodes copy one shared block through the write path + description: Node A commits its copy of a shared block first; node B's commit fails, B frees only its own copy and retries in place, and the old block loses exactly one owner + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestConcurrentCopyOnWriteAcrossNodes + + - id: snapshot-old-image-safe + name: Old snapshot and old clone after upgrading + description: On an image with an older version's snapshot and clone of one file, writes and deleting the snapshot, clone and source never free a block another inode maps + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestOldSnapshotAndCloneNeverFreeSharedBlocks + - id: snapshot-recovery-after-crash name: Snapshot cleanup on mount after a crash description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner @@ -1494,6 +1508,22 @@ scenarios: docker: run-tests.sh::test_snapshot_delete_after_overwrite vm: run-vm-tests.sh::test_snapshot_delete_after_overwrite + - id: snapshot-concurrent-creates-across-nodes + name: Concurrent snapshots on two nodes + description: Two nodes snapshot their own files at the same time; each file keeps all its snapshots, and deleting one node's snapshots leaves the other's + priority: high + environments: + go: snapshot_test.go::TestSnapshotConcurrentCreatesAcrossNodes + docker: run-tests.sh::test_snapshot_concurrent_creates + vm: run-vm-tests.sh::test_snapshot_concurrent_creates + + - id: snapshot-registry-concurrent-creates + name: Registry creates from two nodes at once + description: Two registries with their own lock managers create snapshots concurrently; none share a slot or ID + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryConcurrentCreatesAcrossNodes + - id: clone-from-snapshot name: Clone from snapshot description: Create writable clone from named snapshot via xattr diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 7363633c..28f38e64 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -2991,6 +2991,59 @@ test_snapshot_delete_after_overwrite() { return 0 } +# Two VMs each take 10 snapshots of their own file at the same time. Each file +# must list all of its snapshots, and deleting one VM's snapshots must leave +# the other VM's alone. +test_snapshot_concurrent_creates() { + log_info "Testing concurrent snapshot creation on two VMs..." + ensure_cluster_healthy 2 + + local run=$$ + local file_a=/mnt/goclusterfs/conc_a.bin + local file_b=/mnt/goclusterfs/conc_b.bin + + ssh_run 1 "rm -f $file_a && head -c 1048576 /dev/urandom > $file_a" || return 1 + ssh_run 2 "rm -f $file_b && head -c 1048576 /dev/urandom > $file_b" || return 1 + + ssh_run 1 "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_a_${run}_\$i $file_a || exit 1; done" & + local pid_a=$! + ssh_run 2 "for i in \$(seq 1 10); do setfattr -n user.snapshot -v conc_b_${run}_\$i $file_b || exit 1; done" & + local pid_b=$! + wait "$pid_a" || { log_error "snapshots on VM 1 failed"; return 1; } + wait "$pid_b" || { log_error "snapshots on VM 2 failed"; return 1; } + + local count_a count_b + count_a=$(ssh_run 1 "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -c ':active'") + count_b=$(ssh_run 2 "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_a" -ne 10 ] || [ "$count_b" -ne 10 ]; then + log_error "$count_a and $count_b active snapshots, want 10 each" + return 1 + fi + + ssh_run 1 "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_a_${run}_\$i $file_a || exit 1; done" || { + log_error "deleting snapshots on VM 1 failed" + return 1 + } + local tries=0 + while ssh_run 1 "getfattr --only-values -n user.snapshots $file_a 2>/dev/null | grep -q conc_a_"; do + tries=$((tries + 1)) + if [ "$tries" -ge 100 ]; then + log_error "VM 1 snapshots still listed 10s after delete" + return 1 + fi + sleep 0.1 + done + count_b=$(ssh_run 2 "getfattr --only-values -n user.snapshots $file_b 2>/dev/null | grep -c ':active'") + if [ "$count_b" -ne 10 ]; then + log_error "VM 2 lists $count_b snapshots after VM 1 deleted its own, want 10" + return 1 + fi + + ssh_run 2 "for i in \$(seq 1 10); do setfattr -x user.snapshot.conc_b_${run}_\$i $file_b; done" + log_info "Concurrent snapshot creation test passed" + return 0 +} + # Two VMs append 1MB alternately to their own file, so both files get # non-contiguous blocks and grow past the old 2MB limit. Checks both files on # each VM after restarting it. @@ -3095,6 +3148,7 @@ main() { run_test "Snapshot Preserves Data" test_snapshot_preserves_data run_test "Clone In-Place Write" test_clone_in_place_write run_test "Snapshot Delete After Overwrite" test_snapshot_delete_after_overwrite + run_test "Snapshot Concurrent Creates" test_snapshot_concurrent_creates run_test "Concurrent Writes" test_concurrent_writes run_test "Concurrent File Creation" test_concurrent_file_creation From ec6b52bb17c7cc5d372479afb0cd52221165c075 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 13:32:58 -0500 Subject: [PATCH 10/18] fix(snapshot): never release uncounted blocks, and never roll back active snapshots A copy-on-write of a block an older version shared without counting its owners released it like a counted one. With two old snapshots and two old clones, two copies dropped the count from 2 to 0 and freed a block the other clone mapped. Such blocks are now copied but never released. Mount recovery marked any snapshot it saw creating as deleting, even if another node had activated it since. It now uses a guarded creating-to-deleting transition, and the delete job refuses snapshots that are not marked deleting. Co-Authored-By: Claude Opus 5.5 --- internal/extent/extent.go | 8 +++- internal/goclusterfs/fs.go | 10 +++-- internal/goclusterfs/snapshot_test.go | 53 +++++++++++++++++++++++++++ internal/snapshot/snapshot.go | 6 +++ internal/snapshot/snapshot_test.go | 7 ++++ internal/snapshot/worker.go | 12 ++++-- internal/snapshot/worker_test.go | 30 +++++++++++++++ tests/test-manifest.yaml | 14 +++++++ 8 files changed, 131 insertions(+), 9 deletions(-) diff --git a/internal/extent/extent.go b/internal/extent/extent.go index 9790f681..f40fc395 100644 --- a/internal/extent/extent.go +++ b/internal/extent/extent.go @@ -69,6 +69,9 @@ type CoWResult struct { OldPhysBlock uint64 // Original physical block (for refcount decrement) NewPhysBlock uint64 // New physical block after CoW WasShared bool // True if CoW was actually performed + // Uncounted means an older version shared the old block without counting + // its owners, so it must never be released. + Uncounted bool } // Extent represents a contiguous range of blocks. @@ -557,7 +560,10 @@ func (m *Manager) CopyOnWrite(ctx context.Context, ino *inode.Inode, ext *Extent if err != nil { return nil, err } - result := &CoWResult{OldPhysBlock: cur.PhysicalBlock + offset} + result := &CoWResult{ + OldPhysBlock: cur.PhysicalBlock + offset, + Uncounted: cur.Flags&FlagShared != 0 && cur.Flags&FlagCounted == 0, + } block := Extent{logicalBlock, result.OldPhysBlock, 1, cur.Flags} needsCoW, err := m.NeedsCoW(ctx, &block, refcountReg) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index f9c3fc59..9186f04b 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2716,11 +2716,13 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseInodeLock() - if len(cowDone) > 0 { - copied := make([]uint64, len(cowDone)) - for i, cow := range cowDone { - copied[i] = cow.OldPhysBlock + var copied []uint64 + for _, cow := range cowDone { + if !cow.Uncounted { + copied = append(copied, cow.OldPhysBlock) } + } + if len(copied) > 0 { f.fs.extentMgr.ReleaseBlocks(ctx, copied) } diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 55684bf0..cb3454cb 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -490,3 +490,56 @@ func TestOldSnapshotAndCloneNeverFreeSharedBlocks(t *testing.T) { } stillAllocated("deleting the source") } + +// On an older version's image, file F had two snapshots (refcount 2) and then +// two clones whose owners were never counted. Copying on write must leave the +// old block to the other clone instead of releasing it. +func TestOldSharingCopiesWithoutReleasing(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(8, BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + fIno, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + fIno.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, fIno); err != nil { + t.Fatal(err) + } + old := physicalBlock(t, fsys, f.inodeNum, 0) + if err := fsys.refcountReg.Set(ctx, old, 2, refcount.FlagNone); err != nil { + t.Fatal(err) + } + var clones []*File + for range 2 { + c := newRegularFile(t, fsys) + cIno, err := fsys.inodeTbl.Read(c.inodeNum) + if err != nil { + t.Fatal(err) + } + cIno.InlineExtents, cIno.ExtentCount = fIno.InlineExtents, fIno.ExtentCount + cIno.Size, cIno.AllocatedBlocks, cIno.WrittenBlocks = fIno.Size, fIno.AllocatedBlocks, fIno.WrittenBlocks + if err := fsys.inodeTbl.Write(ctx, cIno); err != nil { + t.Fatal(err) + } + clones = append(clones, c) + } + + writeAt(t, f, pattern(9, 100), 10) + writeAt(t, clones[0], pattern(10, 100), 20) + + if ok, err := fsys.bitmapMgr.IsAllocated(old); err != nil || !ok { + t.Fatalf("old block %d freed while the second clone maps it (err=%v)", old, err) + } + if got := refcountOf(t, fsys, old); got != 2 { + t.Fatalf("old block refcount %d, want 2: copies of uncounted blocks must not release them", got) + } + checkFile(t, clones[1], original) + checkFile(t, f, overwritten(original, 10, pattern(9, 100))) + checkFile(t, clones[0], overwritten(original, 20, pattern(10, 100))) +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 425672ea..580fae10 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -468,6 +468,12 @@ func (r *Registry) Activate(ctx context.Context, snapshotID uint64) error { return r.updateState(ctx, snapshotID, StateActive, StateCreating) } +// Abandon marks a snapshot that is still being created as deleting. It fails +// with ErrSnapshotInvalidState if the snapshot became active meanwhile. +func (r *Registry) Abandon(ctx context.Context, snapshotID uint64) error { + return r.updateState(ctx, snapshotID, StateDeleting, StateCreating) +} + // updateState sets a snapshot's state, requiring it to be in from unless from // is StateInvalid. func (r *Registry) updateState(ctx context.Context, snapshotID uint64, newState, from State) error { diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 5ed8d904..73c1c8df 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -908,6 +908,13 @@ func TestRegistryActivateRequiresCreating(t *testing.T) { if err := registry.Activate(ctx, fresh.SnapshotID); err != nil { t.Fatalf("Activate of a creating snapshot: %v", err) } + // Another node's mount must not roll back a snapshot that became active. + if err := registry.Abandon(ctx, fresh.SnapshotID); !errors.Is(err, ErrSnapshotInvalidState) { + t.Fatalf("Abandon of an active snapshot: got %v, want ErrSnapshotInvalidState", err) + } + if got, err := registry.Get(ctx, fresh.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("snapshot after refused Abandon: %+v, %v; want active", got, err) + } } // Two nodes, each with its own lock manager on the shared lock region, create diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index 7062cba9..ec4b7b26 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -199,6 +199,9 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { if err != nil { return fmt.Errorf("failed to get snapshot %d: %w", job.SnapshotID, err) } + if snap.State != StateDeleting { + return fmt.Errorf("snapshot %d is %s, not deleting: %w", job.SnapshotID, snap.State, ErrSnapshotInvalidState) + } children, err := w.registry.GetChildren(ctx, job.SnapshotID) if err != nil { @@ -289,8 +292,9 @@ func (w *Worker) resultHandler() { } // RecoverIncompleteJobs deletes snapshots that a crash left creating or -// deleting. A snapshot still creating never became visible as active, so it -// is rolled back. Call this on mount, after Start. +// deleting. A snapshot still creating was never reported as created, so it is +// rolled back, unless another node activates it first. Call this on mount, +// after Start. func (w *Worker) RecoverIncompleteJobs(ctx context.Context) error { w.mu.Lock() defer w.mu.Unlock() @@ -304,8 +308,8 @@ func (w *Worker) RecoverIncompleteJobs(ctx context.Context) error { for _, snap := range snapshots { switch snap.State { case StateCreating: - if err := w.registry.Delete(ctx, snap.SnapshotID); err != nil { - log.Printf("[snapshot/worker] Failed to roll back snapshot %d: %v", snap.SnapshotID, err) + if err := w.registry.Abandon(ctx, snap.SnapshotID); err != nil { + log.Printf("[snapshot/worker] Not rolling back snapshot %d: %v", snap.SnapshotID, err) continue } case StateDeleting: diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index 90f3e213..fbda8138 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -624,3 +624,33 @@ func TestMultipleChildrenMergeUp(t *testing.T) { t.Errorf("Middle snapshot should be removed, got: %v", err) } } + +// A delete job for a snapshot that is not marked deleting, such as one another +// node activated after a mount listed it as creating, must leave it alone. +func TestDeleteJobRequiresDeletingState(t *testing.T) { + env := newWorkerEnv(t) + env.start() + file, blocks := env.file(1) + snap := env.snapshot(file, "active", StateActive) + + if _, err := env.w.QueueDeleteJob(env.ctx, snap.SnapshotID); err != nil { + t.Fatal(err) + } + select { + case r := <-env.done: + if r.Success { + t.Fatal("delete job succeeded on an active snapshot") + } + case <-time.After(10 * time.Second): + t.Fatal("delete job did not finish") + } + if got, err := env.reg.Get(env.ctx, snap.SnapshotID); err != nil || got.State != StateActive { + t.Fatalf("snapshot after refused delete: %+v, %v; want active", got, err) + } + if ino, err := env.tbl.Read(snap.DataInode); err != nil || ino.Magic == 0 { + t.Fatalf("data inode %d freed by a refused delete (err=%v)", snap.DataInode, err) + } + if e, err := env.refs.Get(env.ctx, blocks[0]); err != nil || e.RefCount != 2 { + t.Fatalf("block refcount %+v, %v; want 2", e, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index c83abdfb..fe925cb0 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1357,6 +1357,20 @@ scenarios: environments: go: internal/goclusterfs/snapshot_test.go::TestOldSnapshotAndCloneNeverFreeSharedBlocks + - id: snapshot-old-sharing-cow + name: Copy-on-write of blocks an older version shared + description: Writing a file and one of its clones from an older version copies the uncounted block but never releases it, so the other clone keeps its data + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestOldSharingCopiesWithoutReleasing + + - id: snapshot-delete-job-requires-deleting + name: Delete job only for deleting snapshots + description: A delete job for an active snapshot fails and leaves the snapshot, its data inode and refcounts alone + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState + - id: snapshot-recovery-after-crash name: Snapshot cleanup on mount after a crash description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner From e0d77d96a8a40ae21b8c3430e8ce41e3f4c990b5 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:02:21 -0500 Subject: [PATCH 11/18] fix(snapshot): keep snapshot IDs and descriptor sectors apart per node In CI, two nodes creating snapshots at once still got the same ID even under the registry lock, because the cross-node lock does not always exclude them. A new ID is now a counter above the highest ID on disk, times 64, plus the node slot. Each node fills descriptor sectors it alone owns first, so two nodes neither reuse an ID nor overwrite each other's sector. The registry test now runs with no lock at all. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 2 +- internal/goclusterfs/fs.go | 8 ++++--- internal/goclusterfs/snapshot_test.go | 2 +- internal/snapshot/snapshot.go | 30 ++++++++++++++++++++++----- internal/snapshot/snapshot_test.go | 15 +++++++------- tests/test-manifest.yaml | 2 +- 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 405d4ec3..513f9202 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -304,7 +304,7 @@ func main() { // Initialize snapshot subsystem after verifying filesystem is valid if *snapshotsEnabled { - if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize, lockMgr); err != nil { + if err := gfs.InitSnapshots(ctx, snapshotOffset, snapshotSize, lockMgr, slotIdx); err != nil { log.Fatalf("Failed to initialize snapshots: %v", err) } log.Printf("Snapshots: enabled (registry at %dMB, %dMB capacity)", diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 9186f04b..5b846951 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -233,17 +233,19 @@ func NewWithComponents( // Parameters: // - snapshotOffset: byte offset where snapshot region starts on device // - snapshotSize: size of snapshot region in bytes -// - lockMgr: serializes registry changes across nodes, so two nodes never -// take the same descriptor slot or snapshot ID (nil for single-node use) +// - lockMgr: serializes registry changes across nodes (nil for single-node use) +// - nodeSlot: this mount's node registry slot, which keeps snapshot IDs and +// descriptor sectors apart from other nodes' even without the lock // // This must be called before any snapshot operations can succeed. -func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64, lockMgr *lock.Manager) error { +func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uint64, lockMgr *lock.Manager, nodeSlot int) error { if f.refcountReg == nil { return fmt.Errorf("refcount region must be initialized before snapshots") } // Create snapshot registry f.snapshotReg = snapshot.NewRegistry(f.dev, snapshotOffset, snapshotSize, lockMgr) + f.snapshotReg.SetNodeSlot(nodeSlot) // Scan registry to rebuild state stats, err := f.snapshotReg.Scan(ctx) diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index cb3454cb..49ad4757 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -61,7 +61,7 @@ func openSnapshotFS(t *testing.T, path string) *FS { refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, nil) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, nil, snapFSDataOffset) fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, nodeUUID, snapFSDataOffset) - if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, nil); err != nil { + if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, nil, 0); err != nil { t.Fatalf("InitSnapshots: %v", err) } t.Cleanup(fsys.snapshotWkr.Stop) diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 580fae10..dca8b8f6 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -21,6 +21,7 @@ import ( "time" "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -230,10 +231,21 @@ type Registry struct { size uint64 // Region size in bytes maxDescriptors uint64 // Maximum number of descriptors lockMgr *lock.Manager // For distributed locking (optional) + nodeSlot int // This node's slot in the node registry nextID uint64 // Next available snapshot ID (cached) descriptorCount uint64 // Active descriptor count (cached) } +// SetNodeSlot tells the registry which node slot this mount holds. New IDs +// end in the slot and each node fills its own sectors first, so two nodes +// never create the same ID or write the same sector at once, even if the +// registry lock fails to exclude them. +func (r *Registry) SetNodeSlot(slot int) { + r.mu.Lock() + defer r.mu.Unlock() + r.nodeSlot = slot % node.MaxNodes +} + // NewRegistry creates a new snapshot registry manager. // // Parameters: @@ -289,7 +301,8 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, if err != nil { return nil, err } - snapshotID := max(r.nextID, maxID+1) + base := max(maxID, r.nextID-1) + snapshotID := (base/node.MaxNodes+1)*node.MaxNodes + uint64(r.nodeSlot) r.nextID = snapshotID + 1 if dataInode != 0 { @@ -680,11 +693,12 @@ type ScanStats struct { TotalBlocks uint64 // Total blocks referenced by all snapshots } -// freeSlotAndMaxID returns the first free slot and the highest snapshot ID on -// disk, which includes snapshots other nodes created since this node's scan. -// Unreadable slots count as free, as they did before. +// freeSlotAndMaxID returns a free slot, preferring sectors this node owns, and +// the highest snapshot ID on disk, which includes snapshots other nodes +// created since this node's scan. Unreadable slots count as free, as they did +// before. func (r *Registry) freeSlotAndMaxID() (uint64, uint64, error) { - free := r.maxDescriptors + free, owned := r.maxDescriptors, r.maxDescriptors var maxID uint64 for sector := uint64(0); sector*DescriptorsPerSector < r.maxDescriptors; sector++ { data, readErr := r.readSector(sector) @@ -696,11 +710,17 @@ func (r *Registry) freeSlotAndMaxID() (uint64, uint64, error) { desc, err := decodeDescriptor(data, i) if readErr != nil || err != nil || desc.IsFree() { free = min(free, slot) + if sector%node.MaxNodes == uint64(r.nodeSlot) { + owned = min(owned, slot) + } continue } maxID = max(maxID, desc.SnapshotID) } } + if owned < r.maxDescriptors { + return owned, maxID, nil + } if free == r.maxDescriptors { return 0, 0, ErrSnapshotFull } diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 73c1c8df..d40ed01e 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -9,7 +9,6 @@ import ( "sync" "testing" - "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -863,7 +862,8 @@ func TestRegistryIDsUniqueAcrossNodes(t *testing.T) { nodeA := NewRegistry(dev, 0, 10*SectorSize, nil) nodeB := NewRegistry(dev, 0, 10*SectorSize, nil) - for _, r := range []*Registry{nodeA, nodeB} { + for slot, r := range []*Registry{nodeA, nodeB} { + r.SetNodeSlot(slot) if _, err := r.Scan(ctx); err != nil { t.Fatal(err) } @@ -917,8 +917,9 @@ func TestRegistryActivateRequiresCreating(t *testing.T) { } } -// Two nodes, each with its own lock manager on the shared lock region, create -// snapshots at the same time. Every snapshot must keep its own slot and ID. +// Two nodes create snapshots at the same time with no lock between them, as +// when the distributed lock fails to exclude them. Node slots alone must keep +// every snapshot in its own slot with its own ID. func TestRegistryConcurrentCreatesAcrossNodes(t *testing.T) { dev, cleanup := createTestDevice(t, 4<<20) defer cleanup() @@ -927,10 +928,8 @@ func TestRegistryConcurrentCreatesAcrossNodes(t *testing.T) { var nodes []*Registry for slot := range 2 { - var nodeUUID [16]byte - nodeUUID[0] = byte(slot + 1) - locks := lock.NewManager(dev, 1<<20, 1<<20, nodeUUID, slot) - r := NewRegistry(dev, 0, 10*SectorSize, locks) + r := NewRegistry(dev, 0, 10*SectorSize, nil) + r.SetNodeSlot(slot) if _, err := r.Scan(ctx); err != nil { t.Fatal(err) } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index fe925cb0..f55e10bc 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1533,7 +1533,7 @@ scenarios: - id: snapshot-registry-concurrent-creates name: Registry creates from two nodes at once - description: Two registries with their own lock managers create snapshots concurrently; none share a slot or ID + description: Two registries create snapshots concurrently with no lock between them; node slots keep every slot and ID apart priority: go_only environments: go: internal/snapshot/snapshot_test.go::TestRegistryConcurrentCreatesAcrossNodes From 20941cd02ee0cb25b276cf7d85e9790f23b4bf77 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:07:47 -0500 Subject: [PATCH 12/18] test(snapshot): check IDs after older images and node slot reuse Co-Authored-By: Claude Opus 5.5 --- internal/snapshot/snapshot_test.go | 59 ++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 7 ++++ 2 files changed, 66 insertions(+) diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index d40ed01e..deb8903f 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -970,3 +970,62 @@ func TestRegistryConcurrentCreatesAcrossNodes(t *testing.T) { names[d.SnapshotID] = d.GetName() } } + +// Older versions numbered snapshots 1, 2, 3... New IDs must land above every +// one of them, and a node that takes over a departed node's slot must never +// reuse that node's IDs. +func TestRegistryIDsAfterOlderImagesAndSlotReuse(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + ctx := context.Background() + + old := NewRegistry(dev, 0, 10*SectorSize, nil) + for i := range 70 { + if _, err := old.Create(ctx, fmt.Sprintf("old-%d", i), 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly); err != nil { + t.Fatal(err) + } + } + // Rewrite them with the sequential IDs an older version would have assigned. + all, err := old.List(ctx) + if err != nil { + t.Fatal(err) + } + for slot := uint64(0); slot < old.maxDescriptors; slot++ { + d, err := old.readDescriptor(slot) + if err != nil || d.IsFree() { + continue + } + var n int + fmt.Sscanf(d.GetName(), "old-%d", &n) + d.SnapshotID = uint64(n + 1) + if err := old.writeDescriptor(slot, d); err != nil { + t.Fatal(err) + } + } + if len(all) != 70 { + t.Fatalf("%d old snapshots, want 70", len(all)) + } + + create := func(slot int, name string) uint64 { + t.Helper() + r := NewRegistry(dev, 0, 10*SectorSize, nil) + r.SetNodeSlot(slot) + if _, err := r.Scan(ctx); err != nil { + t.Fatal(err) + } + d, err := r.Create(ctx, name, 100, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly) + if err != nil { + t.Fatal(err) + } + return d.SnapshotID + } + first := create(5, "from-slot-5") + if first <= 70 || first%64 != 5 { + t.Fatalf("ID %d after IDs 1-70 from an older image, want above 70 and ending in slot 5", first) + } + // A new node takes over slot 5 after the first owner left. + second := create(5, "slot-5-new-owner") + if second <= first || second%64 != 5 { + t.Fatalf("new owner of slot 5 got ID %d after %d, want a higher ID ending in slot 5", second, first) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index f55e10bc..baefc832 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1385,6 +1385,13 @@ scenarios: environments: go: internal/snapshot/snapshot_test.go::TestRegistryIDsUniqueAcrossNodes + - id: snapshot-ids-after-older-images + name: Snapshot IDs on an older image and after slot reuse + description: New IDs land above the sequential IDs an older version assigned, and a node that takes over a departed node's slot never reuses its IDs + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestRegistryIDsAfterOlderImagesAndSlotReuse + - id: snapshot-activate-requires-creating name: Snapshot activation after a rollback description: A snapshot another node rolled back cannot be marked active From 6f7faffe79837a9c8266162270c0e4a2f1d9d283 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:12:54 -0500 Subject: [PATCH 13/18] fix(snapshot): copy shared blocks before taking the inode lock With #13's FS locks, a write into a shared block held the inode lock and then needed the refcount and bitmap locks, which come first in lock order, so it failed with EIO. The write now persists and drops the inode lock, copies the block on its unlocked copy, then takes the lock and keeps the copy only if no node wrote the inode meanwhile; otherwise it frees the copy and retries. A conflict after an earlier mid-write commit frees only the copies that commit did not map. The in-process snapshot tests now run with a lock manager per FS, like separate nodes. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 134 +++++++++++--------------- internal/goclusterfs/snapshot_test.go | 28 ++++-- 2 files changed, 75 insertions(+), 87 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 5b846951..49d73860 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -42,8 +42,8 @@ import ( // modification was detected and the Write operation should be retried. var errRetryWrite = errors.New("retry write operation") -// beforeCoWCommit lets tests run another node's write between a copy-on-write -// and the inode write that commits it. +// beforeCoWCommit lets tests run another node's write after a copy-on-write, +// before this write takes the inode lock and commits the copy. var beforeCoWCommit func() // MaxWriteRetries is the maximum number of retry attempts for Write operations @@ -2367,76 +2367,55 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } handleExtent: - // === Copy-on-Write for Shared Blocks (Snapshot Support) === - // If this extent is shared with a snapshot (FlagShared + RefCount > 1), - // we must allocate a new block before writing to preserve snapshot data. + // A shared block is copied before the write, so its other owners keep their data. if ext.Flags&extent.FlagShared != 0 && f.fs.refcountReg != nil { - // Check if CoW is actually needed (refcount > 1) + // Refcounts and the bitmap come before the inode in lock order. + if err := persistAndUnlock(); err != nil { + return 0, err + } needsCoW, cowErr := f.fs.extentMgr.NeedsCoW(ctx, blockExtent(ext, extOffset), f.fs.refcountReg) if cowErr != nil { log.Printf("[DEBUG] Write inode %d: CoW check failed for block %d: %v", f.inodeNum, ext.PhysicalBlock, cowErr) return 0, syscall.EIO } - if needsCoW { - // Need exclusive inode lock for CoW (modifies extent tree) - if !lockHeld && f.fs.lockMgr != nil { - if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - log.Printf("[DEBUG] Write inode %d: failed to acquire lock for CoW: %v", f.inodeNum, err) - return 0, syscall.EIO - } - lockHeld = true - - // Re-read inode after acquiring lock - freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: re-read for CoW failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - if freshIno.MetaGeneration != ino.MetaGeneration { - log.Printf("[DEBUG] Write inode %d: MetaGeneration changed during CoW", f.inodeNum) - return 0, errRetryWrite - } - ino = freshIno - - // Re-lookup extent - another node may have already done CoW - ext, extOffset, err = f.fs.extentMgr.Lookup(ino, blockNum) - if err != nil { - log.Printf("[DEBUG] Write inode %d: extent lookup for CoW failed: %v", f.inodeNum, err) - return 0, syscall.EIO - } - - // Re-check if CoW is still needed (another node may have done it) - needsCoW, cowErr = f.fs.extentMgr.NeedsCoW(ctx, blockExtent(ext, extOffset), f.fs.refcountReg) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: CoW re-check failed: %v", f.inodeNum, cowErr) - return 0, syscall.EIO - } + cowResult, cowErr := f.fs.extentMgr.CopyOnWrite(ctx, ino, ext, blockNum, f.fs.dataOffset, f.fs.refcountReg) + if cowErr != nil { + log.Printf("[DEBUG] Write inode %d: CoW failed for block %d: %v", + f.inodeNum, ext.PhysicalBlock, cowErr) + f.fs.extentMgr.DiscardPending(ctx, ino) + return 0, syscall.EIO } - - // Perform CoW if still needed - if needsCoW { - cowResult, cowErr := f.fs.extentMgr.CopyOnWrite(ctx, ino, ext, blockNum, f.fs.dataOffset, f.fs.refcountReg) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: CoW failed for block %d: %v", - f.inodeNum, ext.PhysicalBlock, cowErr) - return 0, syscall.EIO + if cowResult.WasShared { + if beforeCoWCommit != nil { + beforeCoWCommit() } - - if cowResult.WasShared { - inoDirty = true - log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", - f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) - - cowDone = append(cowDone, cowResult) - - // Re-lookup extent to get updated physical block - ext, extOffset, cowErr = f.fs.extentMgr.Lookup(ino, blockNum) - if cowErr != nil { - log.Printf("[DEBUG] Write inode %d: extent lookup after CoW failed: %v", f.inodeNum, cowErr) + // Copied on an unlocked copy: keep it only if no node wrote the inode meanwhile. + if f.fs.lockMgr != nil { + if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + log.Printf("[DEBUG] Write inode %d: failed to acquire lock for CoW: %v", f.inodeNum, err) + f.fs.extentMgr.DiscardPending(ctx, ino) + f.fs.extentMgr.FreePhysicalBlock(ctx, cowResult.NewPhysBlock) return 0, syscall.EIO } + lockHeld = true + freshIno, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil || freshIno.MetaGeneration != ino.MetaGeneration { + log.Printf("[DEBUG] Write inode %d: inode changed during CoW (err=%v)", f.inodeNum, err) + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + f.fs.extentMgr.FreePhysicalBlock(ctx, cowResult.NewPhysBlock) + return 0, errRetryWrite + } + } + inoDirty = true + cowDone = append(cowDone, cowResult) + log.Printf("[DEBUG] Write inode %d: CoW completed, old block %d -> new block %d", + f.inodeNum, cowResult.OldPhysBlock, cowResult.NewPhysBlock) + if ext, extOffset, cowErr = f.fs.extentMgr.Lookup(ino, blockNum); cowErr != nil { + log.Printf("[DEBUG] Write inode %d: extent lookup after CoW failed: %v", f.inodeNum, cowErr) + return 0, syscall.EIO } } } @@ -2652,6 +2631,20 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i ino.Size = newSize ino.Mtime = newMtime + // releaseCopied drops this file's ownership of blocks it copied away from. + // Blocks an older version shared without counting stay allocated. + releaseCopied := func(cows []*extent.CoWResult) { + var old []uint64 + for _, cow := range cows { + if !cow.Uncounted { + old = append(old, cow.OldPhysBlock) + } + } + if len(old) > 0 { + f.fs.extentMgr.ReleaseBlocks(ctx, old) + } + } + // On a tree conflict the blocks allocated since the last mid-write persist are // unreferenced. Close the intent first so journal replay cannot map them again, // then free them. Blocks the persist committed stay mapped on disk. @@ -2683,12 +2676,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } // The retry finds these blocks already copied, so release the old ones now. - for _, cow := range cowDone[:committedCoW] { - if _, err := f.fs.refcountReg.Decrement(ctx, cow.OldPhysBlock); err != nil { - log.Printf("[WARN] Write inode %d: failed to decrement refcount on old block %d: %v", - f.inodeNum, cow.OldPhysBlock, err) - } - } + releaseCopied(cowDone[:committedCoW]) return errRetryWrite } @@ -2697,10 +2685,6 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return 0, writeFailed(err) } } - if len(cowDone) > 0 && beforeCoWCommit != nil { - beforeCoWCommit() - } - // Write inode - use WriteWithLockHeld if we hold the lock if lockHeld { if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { @@ -2718,15 +2702,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i releaseInodeLock() - var copied []uint64 - for _, cow := range cowDone { - if !cow.Uncounted { - copied = append(copied, cow.OldPhysBlock) - } - } - if len(copied) > 0 { - f.fs.extentMgr.ReleaseBlocks(ctx, copied) - } + releaseCopied(cowDone) // Cache the freshly-written inode for subsequent reads f.fs.cacheInode(ino) diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 49ad4757..d29ea544 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "sync/atomic" "syscall" "testing" "time" @@ -16,6 +17,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/dir" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/refcount" "github.com/supporttools/GoClusterFS/internal/snapshot" "github.com/supporttools/GoClusterFS/internal/storage" @@ -27,10 +29,16 @@ const ( snapFSRefcountOffset = 3 << 20 snapFSRegistryOffset = 3<<20 + 64<<10 snapFSRegistrySize = 64 << 10 + snapFSLockOffset = 3<<20 + 256<<10 + snapFSLockSize = 512 << 10 snapFSDataOffset = 4 << 20 snapFSDataBlocks = 60 ) +// snapFSNodes hands each opened FS its own node slot, so instances on one +// image lock against each other like separate nodes. +var snapFSNodes atomic.Int32 + func newSnapshotFSImage(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "fs.img") @@ -43,7 +51,8 @@ func newSnapshotFSImage(t *testing.T) string { return path } -// openSnapshotFS mounts an image with snapshots enabled, as a fresh node would. +// openSnapshotFS mounts an image with snapshots and the distributed locks +// enabled, as a fresh node would. func openSnapshotFS(t *testing.T, path string) *FS { t.Helper() dev, err := storage.Open(path) @@ -52,16 +61,19 @@ func openSnapshotFS(t *testing.T, path string) *FS { } t.Cleanup(func() { dev.Close() }) + slot := int(snapFSNodes.Add(1)-1) % 64 var nodeUUID [16]byte copy(nodeUUID[:], "testnode12345678") - bm := bitmap.NewRegion(dev, snapFSBitmapOffset, snapFSDataBlocks, nil, nodeUUID) - inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, nil, nodeUUID) - extentMgr := extent.NewManager(dev, bm, nil, snapFSDataOffset) + nodeUUID[15] = byte(slot) + locks := lock.NewManager(dev, snapFSLockOffset, snapFSLockSize, nodeUUID, slot) + bm := bitmap.NewRegion(dev, snapFSBitmapOffset, snapFSDataBlocks, locks, nodeUUID) + inodeTbl := inode.NewTable(dev, snapFSInodeOffset, 4096*inode.InodeSize, locks, nodeUUID) + extentMgr := extent.NewManager(dev, bm, locks, snapFSDataOffset) inodeTbl.SetTreeFlusher(extentMgr) - refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, nil) - dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, nil, snapFSDataOffset) - fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, nodeUUID, snapFSDataOffset) - if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, nil, 0); err != nil { + refs := refcount.NewRegion(dev, snapFSRefcountOffset, snapFSDataBlocks, locks) + dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, snapFSDataOffset) + fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, snapFSDataOffset) + if err := fsys.InitSnapshots(context.Background(), snapFSRegistryOffset, snapFSRegistrySize, locks, slot); err != nil { t.Fatalf("InitSnapshots: %v", err) } t.Cleanup(fsys.snapshotWkr.Stop) From 6ccaacd7f716f2488cbdd21490b288e85c8bb7e0 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:22:52 -0500 Subject: [PATCH 14/18] test(snapshot): failed final write after a mid-write copy commit Covers #13's release of committed copies on a failed final inode write: each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/snapshot_test.go | 69 +++++++++++++++++++++++++++ tests/test-manifest.yaml | 7 +++ 2 files changed, 76 insertions(+) diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index d29ea544..92378df8 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "os" "path/filepath" "sync/atomic" @@ -555,3 +556,71 @@ func TestOldSharingCopiesWithoutReleasing(t *testing.T) { checkFile(t, f, overwritten(original, 10, pattern(9, 100))) checkFile(t, clones[0], overwritten(original, 20, pattern(10, 100))) } + +// A write over two shared blocks commits the first copy mid-write. When the +// final inode write then fails, each old block must lose this file's +// ownership exactly once across the failed write and its retry, and blocks an +// older version shared without counting must never lose any. +func TestFailedWriteReleasesCommittedCopiesOnce(t *testing.T) { + for _, legacy := range []bool{false, true} { + t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(1, 2*BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + old := []uint64{physicalBlock(t, fsys, f.inodeNum, 0), physicalBlock(t, fsys, f.inodeNum, 1)} + want := uint32(1) + if legacy { + ino, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + ino.InlineExtents[0].Flags |= extent.FlagShared + if err := fsys.inodeTbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + for _, b := range old { + if err := fsys.refcountReg.Set(ctx, b, 2, refcount.FlagNone); err != nil { + t.Fatal(err) + } + } + want = 2 + } else { + setxattr(t, f, "user.snapshot", "before") + } + + conflicts := 1 + testHookBeforeInodeCommit = func() error { + if conflicts > 0 { + conflicts-- + return extent.ErrTreeConflict + } + return nil + } + t.Cleanup(func() { testHookBeforeInodeCommit = nil }) + updated := pattern(2, 2*BlockSize) + writeAt(t, f, updated, 0) + if conflicts != 0 { + t.Fatal("the final inode write never ran") + } + + for _, b := range old { + if got := refcountOf(t, fsys, b); got != want { + t.Errorf("old block %d refcount %d, want %d", b, got, want) + } + if ok, err := fsys.bitmapMgr.IsAllocated(b); err != nil || !ok { + t.Errorf("old block %d freed (err=%v)", b, err) + } + } + checkFile(t, f, updated) + if !legacy { + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + checkFile(t, lookupFile(t, fsys, "before-clone"), original) + } + }) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index baefc832..41796567 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1371,6 +1371,13 @@ scenarios: environments: go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState + - id: snapshot-failed-write-releases-once + name: Failed write after a mid-write copy commit + description: When the final inode write fails after a mid-write persist committed one copied block, each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestFailedWriteReleasesCommittedCopiesOnce + - id: snapshot-recovery-after-crash name: Snapshot cleanup on mount after a crash description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner From d70af7203e5ce138c33f35790f91d8ee3f3bd222 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:26:25 -0500 Subject: [PATCH 15/18] fix(snapshot): log why a snapshot could not be added to the registry Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 49d73860..c79ecfe8 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3435,6 +3435,7 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { desc, err := f.fs.snapshotReg.Create(ctx, name, f.inodeNum, 0, data.InodeNum, src.AllocatedBlocks, src.Size, snapshot.FlagReadOnly) if err != nil { + log.Printf("[snapshot] Failed to add '%s' to the registry: %v", name, err) f.fs.inodeTbl.Free(ctx, data.InodeNum) if errors.Is(err, snapshot.ErrSnapshotExists) { return syscall.EEXIST From 9b7105bd8dce4c7d325a2bf96c093f723ce683f8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:26:40 -0500 Subject: [PATCH 16/18] test(snapshot): skip the two-node concurrent snapshot test in short mode Two nodes can still both claim one free inode until lock claims are confirmed (delta-lease), and creating 20 snapshots on cold lock records across two nodes hits that often. The Docker and VM versions still run. Co-Authored-By: Claude Opus 5.5 --- tests/integration/snapshot_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/snapshot_test.go b/tests/integration/snapshot_test.go index 43817567..b05459af 100644 --- a/tests/integration/snapshot_test.go +++ b/tests/integration/snapshot_test.go @@ -951,6 +951,9 @@ func TestSnapshotDeleteAfterOverwrite(t *testing.T) { // files at the same time. Each file keeps all its snapshots, and deleting one // node's snapshots leaves the other node's alone. func TestSnapshotConcurrentCreatesAcrossNodes(t *testing.T) { + if testing.Short() { + t.Skip("two nodes can still both claim a free inode until lock claims are confirmed (delta-lease)") + } cluster := NewMultiNodeCluster(t, 2, 50) if err := cluster.FormatAndStartFirst(false); err != nil { t.Fatalf("Failed to format and start first node: %v", err) From 096987fe66ea483cfff459b5f6d5e65898239199 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:32:38 -0500 Subject: [PATCH 17/18] fix(inode): reject stale copies whose inline extents changed on disk The stale-write check compared only extent trees. A copy of an inline inode read before a copy-on-write committed, such as a slow chmod on another node, could write the old mapping back after the old block was released, leaving the snapshot's block undercounted. A plain write now fails with ErrTreeConflict when the inline extents on disk differ from the ones the copy was read with; callers already retry. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/snapshot_test.go | 42 ++++++++++++++++++++++++ internal/inode/inode.go | 12 ++++++- internal/inode/inode_test.go | 46 +++++++++++++++++++++++++++ tests/test-manifest.yaml | 14 ++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index 92378df8..4469e2e8 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -624,3 +624,45 @@ func TestFailedWriteReleasesCommittedCopiesOnce(t *testing.T) { }) } } + +// An inode copy read before a copy-on-write committed, as by a slow chmod on +// another node, must not write the old mapping back: the old block already +// lost this file's ownership, so the snapshot would hold it at an undercount. +func TestStaleCopyCannotUndoCopyOnWrite(t *testing.T) { + fsys := openSnapshotFS(t, newSnapshotFSImage(t)) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + original := pattern(1, BlockSize) + f := newRegularFile(t, fsys) + writeAt(t, f, original, 0) + setxattr(t, f, "user.snapshot", "before") + old := physicalBlock(t, fsys, f.inodeNum, 0) + + stale, err := fsys.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + writeAt(t, f, pattern(2, 100), 10) + stale.Mode = inode.S_IFREG | 0o600 + if err := fsys.inodeTbl.Write(ctx, stale); !errors.Is(err, inode.ErrTreeConflict) { + t.Fatalf("stale copy written over a committed copy-on-write: got %v, want ErrTreeConflict", err) + } + + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_MODE + in.Mode = 0o600 + if errno := f.Setattr(ctx, nil, in, &fuse.AttrOut{}); errno != 0 { + t.Fatalf("chmod: %v", errno) + } + if got := physicalBlock(t, fsys, f.inodeNum, 0); got == old { + t.Fatal("the file maps the snapshot's block again") + } + if got := refcountOf(t, fsys, old); got != 1 { + t.Fatalf("old block refcount %d, want 1", got) + } + checkFile(t, f, overwritten(original, 10, pattern(2, 100))) + setxattr(t, f, "user.snapshot.clone", "before:/before-clone") + checkFile(t, lookupFile(t, fsys, "before-clone"), original) +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 230d3316..46924bba 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -274,6 +274,16 @@ func (t *Table) writeWithTree(inode *Inode) (func(), error) { return committed, nil } +// staleExtents reports whether writing inode would replace extents another +// write changed since inode was read, such as a copy-on-write whose old block +// was already released. +func staleExtents(inode, onDisk *Inode) bool { + if inode.HasExtentTree() || onDisk.HasExtentTree() { + return !sameExtentTree(inode, onDisk) + } + return onDisk.InlineExtents != inode.committedInline +} + func sameExtentTree(a, b *Inode) bool { return a.HasExtentTree() == b.HasExtentTree() && a.ExtentTreeRoot == b.ExtentTreeRoot && a.ExtentTreeGen == b.ExtentTreeGen && a.ExtentDepth == b.ExtentDepth @@ -507,7 +517,7 @@ func (t *Table) writeInodeSector(inode *Inode, checkTree bool) error { start := inodeOffset * InodeSize if checkTree { onDisk, err := deserializeInode(buf.Bytes()[start : start+InodeSize]) - if err == nil && (inode.HasExtentTree() || onDisk.HasExtentTree()) && !sameExtentTree(inode, onDisk) { + if err == nil && staleExtents(inode, onDisk) { return fmt.Errorf("%w: inode %d", ErrTreeConflict, inode.InodeNum) } } diff --git a/internal/inode/inode_test.go b/internal/inode/inode_test.go index 1124eecd..96f2dd20 100644 --- a/internal/inode/inode_test.go +++ b/internal/inode/inode_test.go @@ -985,3 +985,49 @@ func TestStaleCopyCannotReplaceExtentTree(t *testing.T) { t.Fatalf("a rejected write changed the inode: %+v", got) } } + +// A copy read before another write changed its inline extents, for example a +// copy-on-write that already released the old block, must not write the old +// mapping back. +func TestStaleCopyCannotUndoInlineExtentChange(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + var uuid [16]byte + tbl := NewTable(dev, 0, 16*InodeSize, nil, uuid) + ctx := context.Background() + + ino := &Inode{Magic: InodeMagic, InodeNum: 6, LinkCount: 1} + ino.InlineExtents[0] = InlineExtent{LogicalBlock: 0, PhysicalBlock: 9, Length: 1, Flags: 1} + if err := tbl.Write(ctx, ino); err != nil { + t.Fatal(err) + } + stale, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + copied, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + copied.InlineExtents[0].PhysicalBlock = 12 + if err := tbl.Write(ctx, copied); err != nil { + t.Fatal(err) + } + + stale.Mode = S_IFREG | 0o600 + if err := tbl.Write(ctx, stale); !errors.Is(err, ErrTreeConflict) { + t.Fatalf("stale inline copy: got %v, want ErrTreeConflict", err) + } + got, err := tbl.Read(6) + if err != nil { + t.Fatal(err) + } + if got.InlineExtents[0].PhysicalBlock != 12 { + t.Fatalf("stale copy restored block %d", got.InlineExtents[0].PhysicalBlock) + } + // A copy that saw the change may write. + got.Mode = S_IFREG | 0o600 + if err := tbl.Write(ctx, got); err != nil { + t.Fatalf("fresh copy: %v", err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 41796567..499c4658 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1378,6 +1378,20 @@ scenarios: environments: go: internal/goclusterfs/snapshot_test.go::TestFailedWriteReleasesCommittedCopiesOnce + - id: snapshot-stale-copy-after-cow + name: Stale inode copy after a copy-on-write + description: An inode copy read before a copy-on-write committed cannot write the old mapping back; a real chmod retries and keeps the copy + priority: go_only + environments: + go: internal/goclusterfs/snapshot_test.go::TestStaleCopyCannotUndoCopyOnWrite + + - id: inode-stale-inline-copy + name: Stale inline inode copy + description: Writing an inode copy whose inline extents changed on disk since it was read fails with ErrTreeConflict + priority: go_only + environments: + go: internal/inode/inode_test.go::TestStaleCopyCannotUndoInlineExtentChange + - id: snapshot-recovery-after-crash name: Snapshot cleanup on mount after a crash description: A mount removes a snapshot left half created or half deleted, frees its data inode, and the file's blocks drop back to one owner From 1a6db24ef109233590540c0edc4c65163f5b7e81 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:32:38 -0500 Subject: [PATCH 18/18] fix(snapshot): retry copy-on-write on a stale tree and finish interrupted snapshot calls A copy-on-write on an unlocked inode copy can find the tree rewritten by another node; it now retries instead of returning EIO. Snapshot and clone xattrs span several writes, and go-fuse cancels the context when the kernel interrupts a request, which made a contended registry lock fail with EIO. They now run to completion. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index c79ecfe8..140f756d 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -2385,6 +2385,10 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: CoW failed for block %d: %v", f.inodeNum, ext.PhysicalBlock, cowErr) f.fs.extentMgr.DiscardPending(ctx, ino) + // The unlocked copy may point at tree nodes another node has since rewritten. + if errors.Is(cowErr, extent.ErrStaleTree) { + return 0, errRetryWrite + } return 0, syscall.EIO } if cowResult.WasShared { @@ -3227,6 +3231,8 @@ func (f *File) allocate(ctx context.Context, off uint64, size uint64, mode uint3 // Used for snapshot creation via user.snapshot attribute and clone creation via user.clone. func (f *File) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno { defer f.fs.lockInode(f.inodeNum)() + // A snapshot or clone spans several writes; an interrupted caller must not stop it halfway. + ctx = context.WithoutCancel(ctx) switch attr { case "user.snapshot": return f.createSnapshot(ctx, string(data)) @@ -3265,7 +3271,7 @@ func (f *File) Removexattr(ctx context.Context, attr string) syscall.Errno { if snapName == "" { return syscall.EINVAL } - return f.deleteSnapshot(ctx, snapName) + return f.deleteSnapshot(context.WithoutCancel(ctx), snapName) } return syscall.ENODATA