From 253fd013007fccaa372a2df1963b9b1100557eba Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 14:50:01 -0600 Subject: [PATCH 1/2] fix(snapshot): let a second delete job leave a reused data inode alone A job that read the descriptor before another job of the same snapshot freed the data inode could then empty whatever took that slot, such as a new snapshot's data inode. The job now empties the data inode under its lock and only while the descriptor still names it, runs one at a time per node, and succeeds when it finds the snapshot already removed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + internal/inode/inode.go | 41 +++++++++++++++++ internal/snapshot/worker.go | 68 +++++++++++++++++++---------- internal/snapshot/worker_test.go | 75 ++++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 7 +++ 5 files changed, 170 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5fd0868..c198e98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Snapshots and clones never copied on write: block refcounts stopped at 1, so a snapshot kept nothing and a write to a clone changed its source. Each snapshot now keeps the blocks it captured in a hidden inode, a write to a shared block copies it first, and a block is freed only when its last owner lets go. - Deleting a snapshot after its file changed failed with a refcount underflow and left the snapshot deleting forever, because the delete used the file's current blocks. - A snapshot or clone could undo another node's change to the source file made while the file was being marked shared, such as the new size after an append, a chmod, or new times. The mark now holds the inode lock from its read to its write. +- Two delete jobs for one snapshot, from a double delete or a node mounting while another node deleted it, could empty the data of a new snapshot that took the freed inode slot. A delete job now empties a data inode only while the snapshot still names it, and a job that finds the snapshot already deleted succeeds. - Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge. - `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects. - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. diff --git a/internal/inode/inode.go b/internal/inode/inode.go index aafb0f60..060be71c 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -549,6 +549,47 @@ func (t *Table) FreeIf(ctx context.Context, inodeNum uint32, ok func(*Inode) boo return true, nil } +// UpdateIf reads an inode under its lock and, if update accepts it, writes back +// the changes update made before the lock is released. It returns the inode as +// update left it and whether it was written. Blocks the write releases are +// freed after the lock is dropped. +func (t *Table) UpdateIf(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (*Inode, bool, error) { + if uint64(inodeNum) >= t.totalInodes { + return nil, false, ErrInodeOutOfRange + } + committed, inode, written, err := t.updateIfLocked(ctx, inodeNum, update) + if committed != nil { + committed() + } + return inode, written, err +} + +func (t *Table) updateIfLocked(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (func(), *Inode, bool, error) { + if t.lockMgr != nil { + if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inodeNum), lock.LockModeExclusive); err != nil { + return nil, nil, false, fmt.Errorf("failed to acquire inode lock: %w", err) + } + defer t.unlock(lock.LockTypeInode, uint64(inodeNum)) + } + inode, err := t.Read(inodeNum) + if err != nil { + return nil, nil, false, err + } + if ok, err := update(inode); err != nil || !ok { + return nil, inode, false, err + } + unlockSector, err := t.lockSector(inodeNum) + if err != nil { + return nil, inode, false, err + } + defer unlockSector() + + t.mu.Lock() + defer t.mu.Unlock() + committed, err := t.writeWithTree(inode) + return committed, inode, err == nil, err +} + // TotalInodes returns the total number of inodes in the table. func (t *Table) TotalInodes() uint64 { return t.totalInodes diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index ec4b7b26..380f19cd 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -42,6 +42,9 @@ type Worker struct { registry *Registry extentMgr *extent.Manager inodeTable *inode.Table + // deleteMu runs this node's delete jobs one at a time. The cluster inode + // lock does not keep two goroutines of one node apart. + deleteMu sync.Mutex jobQueue chan *Job resultChan chan *JobResult @@ -189,9 +192,15 @@ func (w *Worker) processJob(job *Job) *JobResult { } } +// afterDeleteJobGet lets tests run another delete job of the same snapshot +// after this one read the descriptor. +var afterDeleteJobGet func(snapshotID uint64) + // processDeleteJob re-parents the snapshot's children to its parent, gives // back the blocks it captured, and removes its descriptor. func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { + w.deleteMu.Lock() + defer w.deleteMu.Unlock() snap, err := w.registry.Get(ctx, job.SnapshotID) if errors.Is(err, ErrSnapshotNotFound) { return nil @@ -202,6 +211,9 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { if snap.State != StateDeleting { return fmt.Errorf("snapshot %d is %s, not deleting: %w", job.SnapshotID, snap.State, ErrSnapshotInvalidState) } + if afterDeleteJobGet != nil { + afterDeleteJobGet(job.SnapshotID) + } children, err := w.registry.GetChildren(ctx, job.SnapshotID) if err != nil { @@ -216,12 +228,17 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { } if snap.HasData() { - if err := w.ReleaseData(ctx, snap.DataInode); err != nil { + if err := w.ReleaseData(ctx, snap.SnapshotID, snap.DataInode); err != nil { return fmt.Errorf("failed to release data of snapshot %d: %w", job.SnapshotID, err) } } - if err := w.registry.Remove(ctx, job.SnapshotID); err != nil { + err = w.registry.Remove(ctx, job.SnapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + log.Printf("[snapshot/worker] Snapshot %d was deleted by another job meanwhile", job.SnapshotID) + return nil + } + if err != nil { return fmt.Errorf("failed to remove snapshot: %w", err) } @@ -237,32 +254,39 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { return nil } -// ReleaseData gives back the blocks held by a snapshot's data inode. The empty -// extent map is committed before any block is released, so a crash leaks -// blocks at worst and calling it again is safe. -func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { - const attempts = 5 - for attempt := 1; ; attempt++ { - ino, err := w.inodeTable.Read(num) - if err != nil { - return err +// ReleaseData gives back the blocks held by snapshot snapshotID's data inode +// num. It empties the inode under its lock, and only while the snapshot's +// descriptor still names it: the descriptor goes before the inode is freed, so +// the slot has not been reused. The empty extent map is committed before any +// block is released, so a crash leaks blocks at worst and calling it again is +// safe. +func (w *Worker) ReleaseData(ctx context.Context, snapshotID uint64, num uint32) error { + ino, released, err := w.inodeTable.UpdateIf(ctx, num, func(ino *inode.Inode) (bool, error) { + desc, err := w.registry.Get(ctx, snapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + return false, nil } - if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { - return nil + if err != nil { + return false, err } - if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil { - w.extentMgr.DiscardPending(ctx, ino) - return err + if desc.DataInode != num || ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { + return false, nil } - err = w.inodeTable.Write(ctx, ino) - if err == nil { - return nil + if ino.ExtentCount == 0 && !ino.HasExtentTree() { + return false, nil } - w.extentMgr.DiscardPending(ctx, ino) - if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts { - return err + return true, w.extentMgr.Truncate(ctx, ino, 0) + }) + if err != nil { + if ino != nil { + w.extentMgr.DiscardPending(ctx, ino) } + return err + } + if !released { + log.Printf("[snapshot/worker] Nothing to release in inode %d for snapshot %d: already released or no longer its data", num, snapshotID) } + return nil } // resultHandler processes completed job results. diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index f4681725..ce97ca0f 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path/filepath" + "slices" + "sync/atomic" "testing" "time" @@ -654,3 +656,76 @@ func TestDeleteJobRequiresDeletingState(t *testing.T) { t.Fatalf("block refcount %+v, %v; want 2", e, err) } } + +// Two delete jobs run for one snapshot, as when a node mounts while another +// node deletes it. The second job read the descriptor before the first one +// freed the data inode, which a new file or snapshot then took. The second job +// must leave the new inode alone and finish without error. +func TestSecondDeleteJobLeavesReusedDataInode(t *testing.T) { + for _, reuse := range []string{"file", "snapshot"} { + t.Run(reuse, func(t *testing.T) { + env := newWorkerEnv(t) + file, blocks := env.file(2) + old := env.snapshot(file, "old", StateActive) + if err := env.reg.Delete(env.ctx, old.SnapshotID); err != nil { + t.Fatal(err) + } + job := &Job{SnapshotID: old.SnapshotID} + + // Another node's worker, with its own job lock. + other := NewWorker(env.reg, env.ext, env.tbl, DefaultWorkerConfig()) + paused, resume := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + t.Cleanup(func() { afterDeleteJobGet = nil }) + afterDeleteJobGet = func(uint64) { + if first.CompareAndSwap(false, true) { + close(paused) + <-resume + } + } + second := make(chan error, 1) + go func() { second <- other.processDeleteJob(env.ctx, job) }() + <-paused + if err := env.w.processDeleteJob(env.ctx, job); err != nil { + t.Fatalf("first delete job: %v", err) + } + env.checkGone(old) + + num, want, owners := uint32(0), []uint64(nil), uint32(0) + if reuse == "file" { + var ino *inode.Inode + ino, want = env.file(1) + num = ino.InodeNum + } else { + num, want, owners = env.snapshot(file, "new", StateActive).DataInode, blocks, 2 + } + if num != old.DataInode { + t.Fatalf("the new %s took inode %d, not the freed data inode %d", reuse, num, old.DataInode) + } + + close(resume) + if err := <-second; err != nil { + t.Errorf("second delete job: %v", err) + } + ino, err := env.tbl.Read(num) + if err != nil { + t.Fatal(err) + } + got, err := env.ext.DataBlocks(ino) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, want) { + t.Errorf("the new %s maps blocks %v after the second job, want %v", reuse, got, want) + } + for _, b := range want { + if !env.allocated(b) { + t.Errorf("block %d of the new %s was freed", b, reuse) + } + if e, err := env.refs.GetUnlocked(b); err != nil || owners != 0 && e.RefCount != owners { + t.Errorf("block %d: refcount %d (err=%v), want %d", b, e.RefCount, err, owners) + } + } + }) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 4a0f4380..1ebe8b69 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2502,6 +2502,13 @@ scenarios: environments: go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState + - id: snapshot-second-delete-job-reused-inode + name: A second delete job leaves a reused data inode alone + description: Two delete jobs run for one snapshot; the first frees the data inode and a new file or snapshot takes the slot before the second releases data. The second job leaves the new inode and its block counts intact and succeeds + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestSecondDeleteJobLeavesReusedDataInode + - id: snapshot-failed-write-releases-once name: Failed write after a mid-write copy commit description: When the final inode write fails after a mid-write persist committed one copied block, each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none From f12ab95ceecf9fd3e394cba08bd0cc3395873a0e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 14:52:02 -0600 Subject: [PATCH 2/2] test(snapshot): pass the snapshot ID to ReleaseData in the recovery test Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/snapshot_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index c9dba96c..7e2f090a 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -401,7 +401,7 @@ func TestSnapshotRecoveryAfterCrash(t *testing.T) { if err := fsys.snapshotReg.Delete(ctx, deleting.SnapshotID); err != nil { t.Fatal(err) } - if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.DataInode); err != nil { + if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.SnapshotID, deleting.DataInode); err != nil { t.Fatal(err) }