diff --git a/CHANGELOG.md b/CHANGELOG.md index c5fd0868..c198e98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Snapshots and clones never copied on write: block refcounts stopped at 1, so a snapshot kept nothing and a write to a clone changed its source. Each snapshot now keeps the blocks it captured in a hidden inode, a write to a shared block copies it first, and a block is freed only when its last owner lets go. - Deleting a snapshot after its file changed failed with a refcount underflow and left the snapshot deleting forever, because the delete used the file's current blocks. - A snapshot or clone could undo another node's change to the source file made while the file was being marked shared, such as the new size after an append, a chmod, or new times. The mark now holds the inode lock from its read to its write. +- Two delete jobs for one snapshot, from a double delete or a node mounting while another node deleted it, could empty the data of a new snapshot that took the freed inode slot. A delete job now empties a data inode only while the snapshot still names it, and a job that finds the snapshot already deleted succeeds. - Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge. - `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects. - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index c9dba96c..7e2f090a 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -401,7 +401,7 @@ func TestSnapshotRecoveryAfterCrash(t *testing.T) { if err := fsys.snapshotReg.Delete(ctx, deleting.SnapshotID); err != nil { t.Fatal(err) } - if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.DataInode); err != nil { + if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.SnapshotID, deleting.DataInode); err != nil { t.Fatal(err) } diff --git a/internal/inode/inode.go b/internal/inode/inode.go index aafb0f60..060be71c 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -549,6 +549,47 @@ func (t *Table) FreeIf(ctx context.Context, inodeNum uint32, ok func(*Inode) boo return true, nil } +// UpdateIf reads an inode under its lock and, if update accepts it, writes back +// the changes update made before the lock is released. It returns the inode as +// update left it and whether it was written. Blocks the write releases are +// freed after the lock is dropped. +func (t *Table) UpdateIf(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (*Inode, bool, error) { + if uint64(inodeNum) >= t.totalInodes { + return nil, false, ErrInodeOutOfRange + } + committed, inode, written, err := t.updateIfLocked(ctx, inodeNum, update) + if committed != nil { + committed() + } + return inode, written, err +} + +func (t *Table) updateIfLocked(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (func(), *Inode, bool, error) { + if t.lockMgr != nil { + if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inodeNum), lock.LockModeExclusive); err != nil { + return nil, nil, false, fmt.Errorf("failed to acquire inode lock: %w", err) + } + defer t.unlock(lock.LockTypeInode, uint64(inodeNum)) + } + inode, err := t.Read(inodeNum) + if err != nil { + return nil, nil, false, err + } + if ok, err := update(inode); err != nil || !ok { + return nil, inode, false, err + } + unlockSector, err := t.lockSector(inodeNum) + if err != nil { + return nil, inode, false, err + } + defer unlockSector() + + t.mu.Lock() + defer t.mu.Unlock() + committed, err := t.writeWithTree(inode) + return committed, inode, err == nil, err +} + // TotalInodes returns the total number of inodes in the table. func (t *Table) TotalInodes() uint64 { return t.totalInodes diff --git a/internal/snapshot/worker.go b/internal/snapshot/worker.go index ec4b7b26..380f19cd 100644 --- a/internal/snapshot/worker.go +++ b/internal/snapshot/worker.go @@ -42,6 +42,9 @@ type Worker struct { registry *Registry extentMgr *extent.Manager inodeTable *inode.Table + // deleteMu runs this node's delete jobs one at a time. The cluster inode + // lock does not keep two goroutines of one node apart. + deleteMu sync.Mutex jobQueue chan *Job resultChan chan *JobResult @@ -189,9 +192,15 @@ func (w *Worker) processJob(job *Job) *JobResult { } } +// afterDeleteJobGet lets tests run another delete job of the same snapshot +// after this one read the descriptor. +var afterDeleteJobGet func(snapshotID uint64) + // processDeleteJob re-parents the snapshot's children to its parent, gives // back the blocks it captured, and removes its descriptor. func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { + w.deleteMu.Lock() + defer w.deleteMu.Unlock() snap, err := w.registry.Get(ctx, job.SnapshotID) if errors.Is(err, ErrSnapshotNotFound) { return nil @@ -202,6 +211,9 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { if snap.State != StateDeleting { return fmt.Errorf("snapshot %d is %s, not deleting: %w", job.SnapshotID, snap.State, ErrSnapshotInvalidState) } + if afterDeleteJobGet != nil { + afterDeleteJobGet(job.SnapshotID) + } children, err := w.registry.GetChildren(ctx, job.SnapshotID) if err != nil { @@ -216,12 +228,17 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { } if snap.HasData() { - if err := w.ReleaseData(ctx, snap.DataInode); err != nil { + if err := w.ReleaseData(ctx, snap.SnapshotID, snap.DataInode); err != nil { return fmt.Errorf("failed to release data of snapshot %d: %w", job.SnapshotID, err) } } - if err := w.registry.Remove(ctx, job.SnapshotID); err != nil { + err = w.registry.Remove(ctx, job.SnapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + log.Printf("[snapshot/worker] Snapshot %d was deleted by another job meanwhile", job.SnapshotID) + return nil + } + if err != nil { return fmt.Errorf("failed to remove snapshot: %w", err) } @@ -237,32 +254,39 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error { return nil } -// ReleaseData gives back the blocks held by a snapshot's data inode. The empty -// extent map is committed before any block is released, so a crash leaks -// blocks at worst and calling it again is safe. -func (w *Worker) ReleaseData(ctx context.Context, num uint32) error { - const attempts = 5 - for attempt := 1; ; attempt++ { - ino, err := w.inodeTable.Read(num) - if err != nil { - return err +// ReleaseData gives back the blocks held by snapshot snapshotID's data inode +// num. It empties the inode under its lock, and only while the snapshot's +// descriptor still names it: the descriptor goes before the inode is freed, so +// the slot has not been reused. The empty extent map is committed before any +// block is released, so a crash leaks blocks at worst and calling it again is +// safe. +func (w *Worker) ReleaseData(ctx context.Context, snapshotID uint64, num uint32) error { + ino, released, err := w.inodeTable.UpdateIf(ctx, num, func(ino *inode.Inode) (bool, error) { + desc, err := w.registry.Get(ctx, snapshotID) + if errors.Is(err, ErrSnapshotNotFound) { + return false, nil } - if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { - return nil + if err != nil { + return false, err } - if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil { - w.extentMgr.DiscardPending(ctx, ino) - return err + if desc.DataInode != num || ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 { + return false, nil } - err = w.inodeTable.Write(ctx, ino) - if err == nil { - return nil + if ino.ExtentCount == 0 && !ino.HasExtentTree() { + return false, nil } - w.extentMgr.DiscardPending(ctx, ino) - if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts { - return err + return true, w.extentMgr.Truncate(ctx, ino, 0) + }) + if err != nil { + if ino != nil { + w.extentMgr.DiscardPending(ctx, ino) } + return err + } + if !released { + log.Printf("[snapshot/worker] Nothing to release in inode %d for snapshot %d: already released or no longer its data", num, snapshotID) } + return nil } // resultHandler processes completed job results. diff --git a/internal/snapshot/worker_test.go b/internal/snapshot/worker_test.go index f4681725..ce97ca0f 100644 --- a/internal/snapshot/worker_test.go +++ b/internal/snapshot/worker_test.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path/filepath" + "slices" + "sync/atomic" "testing" "time" @@ -654,3 +656,76 @@ func TestDeleteJobRequiresDeletingState(t *testing.T) { t.Fatalf("block refcount %+v, %v; want 2", e, err) } } + +// Two delete jobs run for one snapshot, as when a node mounts while another +// node deletes it. The second job read the descriptor before the first one +// freed the data inode, which a new file or snapshot then took. The second job +// must leave the new inode alone and finish without error. +func TestSecondDeleteJobLeavesReusedDataInode(t *testing.T) { + for _, reuse := range []string{"file", "snapshot"} { + t.Run(reuse, func(t *testing.T) { + env := newWorkerEnv(t) + file, blocks := env.file(2) + old := env.snapshot(file, "old", StateActive) + if err := env.reg.Delete(env.ctx, old.SnapshotID); err != nil { + t.Fatal(err) + } + job := &Job{SnapshotID: old.SnapshotID} + + // Another node's worker, with its own job lock. + other := NewWorker(env.reg, env.ext, env.tbl, DefaultWorkerConfig()) + paused, resume := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + t.Cleanup(func() { afterDeleteJobGet = nil }) + afterDeleteJobGet = func(uint64) { + if first.CompareAndSwap(false, true) { + close(paused) + <-resume + } + } + second := make(chan error, 1) + go func() { second <- other.processDeleteJob(env.ctx, job) }() + <-paused + if err := env.w.processDeleteJob(env.ctx, job); err != nil { + t.Fatalf("first delete job: %v", err) + } + env.checkGone(old) + + num, want, owners := uint32(0), []uint64(nil), uint32(0) + if reuse == "file" { + var ino *inode.Inode + ino, want = env.file(1) + num = ino.InodeNum + } else { + num, want, owners = env.snapshot(file, "new", StateActive).DataInode, blocks, 2 + } + if num != old.DataInode { + t.Fatalf("the new %s took inode %d, not the freed data inode %d", reuse, num, old.DataInode) + } + + close(resume) + if err := <-second; err != nil { + t.Errorf("second delete job: %v", err) + } + ino, err := env.tbl.Read(num) + if err != nil { + t.Fatal(err) + } + got, err := env.ext.DataBlocks(ino) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, want) { + t.Errorf("the new %s maps blocks %v after the second job, want %v", reuse, got, want) + } + for _, b := range want { + if !env.allocated(b) { + t.Errorf("block %d of the new %s was freed", b, reuse) + } + if e, err := env.refs.GetUnlocked(b); err != nil || owners != 0 && e.RefCount != owners { + t.Errorf("block %d: refcount %d (err=%v), want %d", b, e.RefCount, err, owners) + } + } + }) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 4a0f4380..1ebe8b69 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2502,6 +2502,13 @@ scenarios: environments: go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState + - id: snapshot-second-delete-job-reused-inode + name: A second delete job leaves a reused data inode alone + description: Two delete jobs run for one snapshot; the first frees the data inode and a new file or snapshot takes the slot before the second releases data. The second job leaves the new inode and its block counts intact and succeeds + priority: go_only + environments: + go: internal/snapshot/worker_test.go::TestSecondDeleteJobLeavesReusedDataInode + - id: snapshot-failed-write-releases-once name: Failed write after a mid-write copy commit description: When the final inode write fails after a mid-write persist committed one copied block, each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none