Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
- Snapshots and clones never copied on write: block refcounts stopped at 1, so a snapshot kept nothing and a write to a clone changed its source. Each snapshot now keeps the blocks it captured in a hidden inode, a write to a shared block copies it first, and a block is freed only when its last owner lets go.
- Deleting a snapshot after its file changed failed with a refcount underflow and left the snapshot deleting forever, because the delete used the file's current blocks.
- A snapshot or clone could undo another node's change to the source file made while the file was being marked shared, such as the new size after an append, a chmod, or new times. The mark now holds the inode lock from its read to its write.
- Two delete jobs for one snapshot, from a double delete or a node mounting while another node deleted it, could empty the data of a new snapshot that took the freed inode slot. A delete job now empties a data inode only while the snapshot still names it, and a job that finds the snapshot already deleted succeeds.
- Writes failed with `EAGAIN` once a file needed a third extent, so most files over 2 MB could not be written. Extents past the first two now go in an on-disk extent tree, and adjacent extents merge.
- `goclusterfs-fsck` phase 4 never checked file extents, because it read inodes into unaligned buffers that `O_DIRECT` rejects.
- A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk.
Expand Down
2 changes: 1 addition & 1 deletion internal/goclusterfs/snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ func TestSnapshotRecoveryAfterCrash(t *testing.T) {
if err := fsys.snapshotReg.Delete(ctx, deleting.SnapshotID); err != nil {
t.Fatal(err)
}
if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.DataInode); err != nil {
if err := fsys.snapshotWkr.ReleaseData(ctx, deleting.SnapshotID, deleting.DataInode); err != nil {
t.Fatal(err)
}

Expand Down
41 changes: 41 additions & 0 deletions internal/inode/inode.go
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,47 @@ func (t *Table) FreeIf(ctx context.Context, inodeNum uint32, ok func(*Inode) boo
return true, nil
}

// UpdateIf reads an inode under its lock and, if update accepts it, writes back
// the changes update made before the lock is released. It returns the inode as
// update left it and whether it was written. Blocks the write releases are
// freed after the lock is dropped.
func (t *Table) UpdateIf(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (*Inode, bool, error) {
if uint64(inodeNum) >= t.totalInodes {
return nil, false, ErrInodeOutOfRange
}
committed, inode, written, err := t.updateIfLocked(ctx, inodeNum, update)
if committed != nil {
committed()
}
return inode, written, err
}

func (t *Table) updateIfLocked(ctx context.Context, inodeNum uint32, update func(*Inode) (bool, error)) (func(), *Inode, bool, error) {
if t.lockMgr != nil {
if err := t.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(inodeNum), lock.LockModeExclusive); err != nil {
return nil, nil, false, fmt.Errorf("failed to acquire inode lock: %w", err)
}
defer t.unlock(lock.LockTypeInode, uint64(inodeNum))
}
inode, err := t.Read(inodeNum)
if err != nil {
return nil, nil, false, err
}
if ok, err := update(inode); err != nil || !ok {
return nil, inode, false, err
}
unlockSector, err := t.lockSector(inodeNum)
if err != nil {
return nil, inode, false, err
}
defer unlockSector()

t.mu.Lock()
defer t.mu.Unlock()
committed, err := t.writeWithTree(inode)
return committed, inode, err == nil, err
}

// TotalInodes returns the total number of inodes in the table.
func (t *Table) TotalInodes() uint64 {
return t.totalInodes
Expand Down
68 changes: 46 additions & 22 deletions internal/snapshot/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ type Worker struct {
registry *Registry
extentMgr *extent.Manager
inodeTable *inode.Table
// deleteMu runs this node's delete jobs one at a time. The cluster inode
// lock does not keep two goroutines of one node apart.
deleteMu sync.Mutex

jobQueue chan *Job
resultChan chan *JobResult
Expand Down Expand Up @@ -189,9 +192,15 @@ func (w *Worker) processJob(job *Job) *JobResult {
}
}

// afterDeleteJobGet lets tests run another delete job of the same snapshot
// after this one read the descriptor.
var afterDeleteJobGet func(snapshotID uint64)

// processDeleteJob re-parents the snapshot's children to its parent, gives
// back the blocks it captured, and removes its descriptor.
func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error {
w.deleteMu.Lock()
defer w.deleteMu.Unlock()
snap, err := w.registry.Get(ctx, job.SnapshotID)
if errors.Is(err, ErrSnapshotNotFound) {
return nil
Expand All @@ -202,6 +211,9 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error {
if snap.State != StateDeleting {
return fmt.Errorf("snapshot %d is %s, not deleting: %w", job.SnapshotID, snap.State, ErrSnapshotInvalidState)
}
if afterDeleteJobGet != nil {
afterDeleteJobGet(job.SnapshotID)
}

children, err := w.registry.GetChildren(ctx, job.SnapshotID)
if err != nil {
Expand All @@ -216,12 +228,17 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error {
}

if snap.HasData() {
if err := w.ReleaseData(ctx, snap.DataInode); err != nil {
if err := w.ReleaseData(ctx, snap.SnapshotID, snap.DataInode); err != nil {
return fmt.Errorf("failed to release data of snapshot %d: %w", job.SnapshotID, err)
}
}

if err := w.registry.Remove(ctx, job.SnapshotID); err != nil {
err = w.registry.Remove(ctx, job.SnapshotID)
if errors.Is(err, ErrSnapshotNotFound) {
log.Printf("[snapshot/worker] Snapshot %d was deleted by another job meanwhile", job.SnapshotID)
return nil
}
if err != nil {
return fmt.Errorf("failed to remove snapshot: %w", err)
}

Expand All @@ -237,32 +254,39 @@ func (w *Worker) processDeleteJob(ctx context.Context, job *Job) error {
return nil
}

// ReleaseData gives back the blocks held by a snapshot's data inode. The empty
// extent map is committed before any block is released, so a crash leaks
// blocks at worst and calling it again is safe.
func (w *Worker) ReleaseData(ctx context.Context, num uint32) error {
const attempts = 5
for attempt := 1; ; attempt++ {
ino, err := w.inodeTable.Read(num)
if err != nil {
return err
// ReleaseData gives back the blocks held by snapshot snapshotID's data inode
// num. It empties the inode under its lock, and only while the snapshot's
// descriptor still names it: the descriptor goes before the inode is freed, so
// the slot has not been reused. The empty extent map is committed before any
// block is released, so a crash leaks blocks at worst and calling it again is
// safe.
func (w *Worker) ReleaseData(ctx context.Context, snapshotID uint64, num uint32) error {
ino, released, err := w.inodeTable.UpdateIf(ctx, num, func(ino *inode.Inode) (bool, error) {
desc, err := w.registry.Get(ctx, snapshotID)
if errors.Is(err, ErrSnapshotNotFound) {
return false, nil
}
if ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 {
return nil
if err != nil {
return false, err
}
if err := w.extentMgr.Truncate(ctx, ino, 0); err != nil {
w.extentMgr.DiscardPending(ctx, ino)
return err
if desc.DataInode != num || ino.Magic == 0 || ino.Flags&inode.FlagSnapshotData == 0 {
return false, nil
}
err = w.inodeTable.Write(ctx, ino)
if err == nil {
return nil
if ino.ExtentCount == 0 && !ino.HasExtentTree() {
return false, nil
}
w.extentMgr.DiscardPending(ctx, ino)
if !errors.Is(err, inode.ErrTreeConflict) || attempt == attempts {
return err
return true, w.extentMgr.Truncate(ctx, ino, 0)
})
if err != nil {
if ino != nil {
w.extentMgr.DiscardPending(ctx, ino)
}
return err
}
if !released {
log.Printf("[snapshot/worker] Nothing to release in inode %d for snapshot %d: already released or no longer its data", num, snapshotID)
}
return nil
}

// resultHandler processes completed job results.
Expand Down
75 changes: 75 additions & 0 deletions internal/snapshot/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"sync/atomic"
"testing"
"time"

Expand Down Expand Up @@ -654,3 +656,76 @@ func TestDeleteJobRequiresDeletingState(t *testing.T) {
t.Fatalf("block refcount %+v, %v; want 2", e, err)
}
}

// Two delete jobs run for one snapshot, as when a node mounts while another
// node deletes it. The second job read the descriptor before the first one
// freed the data inode, which a new file or snapshot then took. The second job
// must leave the new inode alone and finish without error.
func TestSecondDeleteJobLeavesReusedDataInode(t *testing.T) {
for _, reuse := range []string{"file", "snapshot"} {
t.Run(reuse, func(t *testing.T) {
env := newWorkerEnv(t)
file, blocks := env.file(2)
old := env.snapshot(file, "old", StateActive)
if err := env.reg.Delete(env.ctx, old.SnapshotID); err != nil {
t.Fatal(err)
}
job := &Job{SnapshotID: old.SnapshotID}

// Another node's worker, with its own job lock.
other := NewWorker(env.reg, env.ext, env.tbl, DefaultWorkerConfig())
paused, resume := make(chan struct{}), make(chan struct{})
var first atomic.Bool
t.Cleanup(func() { afterDeleteJobGet = nil })
afterDeleteJobGet = func(uint64) {
if first.CompareAndSwap(false, true) {
close(paused)
<-resume
}
}
second := make(chan error, 1)
go func() { second <- other.processDeleteJob(env.ctx, job) }()
<-paused
if err := env.w.processDeleteJob(env.ctx, job); err != nil {
t.Fatalf("first delete job: %v", err)
}
env.checkGone(old)

num, want, owners := uint32(0), []uint64(nil), uint32(0)
if reuse == "file" {
var ino *inode.Inode
ino, want = env.file(1)
num = ino.InodeNum
} else {
num, want, owners = env.snapshot(file, "new", StateActive).DataInode, blocks, 2
}
if num != old.DataInode {
t.Fatalf("the new %s took inode %d, not the freed data inode %d", reuse, num, old.DataInode)
}

close(resume)
if err := <-second; err != nil {
t.Errorf("second delete job: %v", err)
}
ino, err := env.tbl.Read(num)
if err != nil {
t.Fatal(err)
}
got, err := env.ext.DataBlocks(ino)
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, want) {
t.Errorf("the new %s maps blocks %v after the second job, want %v", reuse, got, want)
}
for _, b := range want {
if !env.allocated(b) {
t.Errorf("block %d of the new %s was freed", b, reuse)
}
if e, err := env.refs.GetUnlocked(b); err != nil || owners != 0 && e.RefCount != owners {
t.Errorf("block %d: refcount %d (err=%v), want %d", b, e.RefCount, err, owners)
}
}
})
}
}
7 changes: 7 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2502,6 +2502,13 @@ scenarios:
environments:
go: internal/snapshot/worker_test.go::TestDeleteJobRequiresDeletingState

- id: snapshot-second-delete-job-reused-inode
name: A second delete job leaves a reused data inode alone
description: Two delete jobs run for one snapshot; the first frees the data inode and a new file or snapshot takes the slot before the second releases data. The second job leaves the new inode and its block counts intact and succeeds
priority: go_only
environments:
go: internal/snapshot/worker_test.go::TestSecondDeleteJobLeavesReusedDataInode

- id: snapshot-failed-write-releases-once
name: Failed write after a mid-write copy commit
description: When the final inode write fails after a mid-write persist committed one copied block, each old block loses the file's ownership once across the retry, and uncounted blocks from an older version lose none
Expand Down
Loading