diff --git a/internal/fsck/fsck.go b/internal/fsck/fsck.go index 551b87d8..7d0a8ffe 100644 --- a/internal/fsck/fsck.go +++ b/internal/fsck/fsck.go @@ -396,6 +396,7 @@ type Checker struct { unnamed []uint32 // inodes other than root that no directory names snapshotInodes map[uint32]bool // inodes referenced by snapshots snapshotData map[uint32]bool // inodes holding a snapshot's captured extents + orphanData map[uint32]int // phase 7's problem for each data inode no snapshot refers to snapsIncomplete []string // why snapshotInodes may miss references sfbContainers map[uint64]bool // blocks holding small files of older versions extentTrees map[uint32]*extentTree @@ -478,6 +479,7 @@ func New(dev *storage.Device, opts Options) *Checker { dirNames: make(map[uint32]int), snapshotInodes: make(map[uint32]bool), snapshotData: make(map[uint32]bool), + orphanData: make(map[uint32]int), sfbContainers: make(map[uint64]bool), extentTrees: make(map[uint32]*extentTree), uncounted: make(map[uint32]int), diff --git a/internal/fsck/phase7_snapshots.go b/internal/fsck/phase7_snapshots.go index b3722503..d3483532 100644 --- a/internal/fsck/phase7_snapshots.go +++ b/internal/fsck/phase7_snapshots.go @@ -207,8 +207,9 @@ func (c *Checker) checkSnapshots() error { for num := range c.snapshotData { if !dataInodes[num] { c.addProblem(PhaseSnapshots, Warning, fmt.Sprintf("inode %d", num), - "snapshot data inode belongs to no snapshot (left by a crash during snapshot creation)", - false) + "snapshot data inode belongs to no snapshot, as a crash while a snapshot is created or deleted leaves it", + true) + c.orphanData[num] = c.lastProblem() } } diff --git a/internal/fsck/repair.go b/internal/fsck/repair.go index e3b211de..4d1c03af 100644 --- a/internal/fsck/repair.go +++ b/internal/fsck/repair.go @@ -23,6 +23,17 @@ func (c *Checker) lastProblem() int { return len(c.result.Problems) - 1 } +// setUnrepairable marks the problem recorded for num, if any, as one repair cannot fix. +func (c *Checker) setUnrepairable(problems map[uint32]int, num uint32) { + i, ok := problems[num] + if !ok { + return + } + c.result.mu.Lock() + defer c.result.mu.Unlock() + c.result.Problems[i].Repairable = false +} + // write writes a repaired sector and marks problems repaired. A dry run only // marks them as ones repair would fix. func (c *Checker) write(buf []byte, offset uint64, problems ...int) error { diff --git a/internal/fsck/unnamed.go b/internal/fsck/unnamed.go index 11b8196b..89fdbb3d 100644 --- a/internal/fsck/unnamed.go +++ b/internal/fsck/unnamed.go @@ -70,12 +70,18 @@ func (c *Checker) checkUnnamed() error { } else { why = c.keepUnnamed(num, ino) } + if why != "" || !reclaim { + c.setUnrepairable(c.orphanData, num) + } if why != "" { if kept++; kept <= reportedLeaks { c.addProblem(PhaseCrossValidation, Warning, loc, "unnamed: no directory names it; kept, because "+why, false) } continue } + if i, ok := c.orphanData[num]; ok { + problems = append(problems, i) + } if free = append(free, ino); len(free) <= reportedLeaks { c.addProblem(PhaseCrossValidation, Warning, loc, "unnamed: no directory names it", reclaim) problems = append(problems, c.lastProblem()) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 3e530d62..10f95557 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3295,13 +3295,13 @@ func sharingErrno(err error) syscall.Errno { // claimInode allocates an inode slot, retrying when another node takes the // free slot first. -func (f *FS) claimInode(ctx context.Context) (*inode.Inode, error) { +func (f *FS) claimInode(ctx context.Context, init func(*inode.Inode)) (*inode.Inode, error) { for attempt := 0; ; attempt++ { num, err := f.inodeTbl.FindFreeSlot() if err != nil { return nil, err } - ino, err := f.inodeTbl.ClaimSlot(ctx, num) + ino, err := f.inodeTbl.ClaimSlotAs(ctx, num, init) if !errors.Is(err, inode.ErrInodeInUse) || attempt == maxConflictRetries-1 { return ino, err } @@ -3364,19 +3364,15 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EIO } - data, err := f.fs.claimInode(ctx) + data, err := f.fs.claimInode(ctx, func(ino *inode.Inode) { + ino.Mode, ino.UID, ino.GID = src.Mode, src.UID, src.GID + ino.Mtime, ino.Ctime = src.Mtime, src.Ctime + ino.Flags |= inode.FlagSnapshotData + }) if err != nil { log.Printf("[snapshot] No inode for the data of a snapshot of inode %d: %v", f.inodeNum, err) return claimErrno(err) } - data.Mode, data.UID, data.GID = src.Mode, src.UID, src.GID - data.Mtime, data.Ctime = src.Mtime, src.Ctime - data.Flags |= inode.FlagSnapshotData - if err := f.fs.inodeTbl.Write(ctx, data); err != nil { - f.fs.freeNewInode(ctx, data.InodeNum) - log.Printf("[snapshot] Failed to write data inode %d: %v", data.InodeNum, err) - return syscall.EIO - } desc, err := f.fs.snapshotReg.Create(ctx, name, f.inodeNum, 0, data.InodeNum, src.AllocatedBlocks, src.Size, snapshot.FlagReadOnly) @@ -3486,22 +3482,18 @@ func (f *File) cloneInode(ctx context.Context, srcNum uint32, destPath string, s return syscall.EIO } - cloneIno, err := f.fs.claimInode(ctx) + now := time.Now().UnixNano() + cloneIno, err := f.fs.claimInode(ctx, func(ino *inode.Inode) { + ino.Mode, ino.UID, ino.GID = srcIno.Mode, srcIno.UID, srcIno.GID + ino.Atime, ino.Mtime, ino.Ctime = now, now, now + }) if err != nil { log.Printf("[clone] Failed to claim an inode slot: %v", err) return claimErrno(err) } cloneInodeNum := cloneIno.InodeNum - - now := time.Now().UnixNano() - cloneIno.Mode = srcIno.Mode - cloneIno.UID = srcIno.UID - cloneIno.GID = srcIno.GID cloneIno.LinkCount = 1 cloneIno.Flags |= inode.FlagHasSnapshot - cloneIno.Atime = now - cloneIno.Mtime = now - cloneIno.Ctime = now if err := f.fs.shareInto(ctx, srcNum, cloneIno, stillValid); err != nil { f.fs.discardInode(ctx, cloneInodeNum) diff --git a/internal/goclusterfs/fsck_test.go b/internal/goclusterfs/fsck_test.go index 823daaa6..c10f838c 100644 --- a/internal/goclusterfs/fsck_test.go +++ b/internal/goclusterfs/fsck_test.go @@ -57,7 +57,25 @@ func newFormattedImageOfSize(t *testing.T, size int64) string { // goclusterfs-mount does. func openFormattedFS(t *testing.T, path string) *FS { t.Helper() - ctx := context.Background() + fsys := openFormattedComponents(t, path) + startSnapshots(t, fsys) + return fsys +} + +func startSnapshots(t *testing.T, fsys *FS) { + t.Helper() + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + if err := fsys.InitSnapshots(context.Background(), sb.SnapshotRegOffset, sb.SnapshotRegSize, fsys.lockMgr, int(fsys.nodeUUID[15])); err != nil { + t.Fatalf("InitSnapshots: %v", err) + } + t.Cleanup(fsys.snapshotWkr.Stop) +} + +func openFormattedComponents(t *testing.T, path string) *FS { + t.Helper() dev, err := storage.Open(path) if err != nil { t.Fatal(err) @@ -79,12 +97,7 @@ func openFormattedFS(t *testing.T, path string) *FS { inodeTbl.SetTreeFlusher(extentMgr) refs := refcount.NewRegion(dev, sb.RefCountTableOffset, sb.RefCountTableSize, sb.TotalBlocks, locks) dirMgr := dir.NewManager(dev, inodeTbl, extentMgr, locks, sb.DataRegionOffset) - fsys := NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, sb.DataRegionOffset) - if err := fsys.InitSnapshots(ctx, sb.SnapshotRegOffset, sb.SnapshotRegSize, locks, slot); err != nil { - t.Fatalf("InitSnapshots: %v", err) - } - t.Cleanup(fsys.snapshotWkr.Stop) - return fsys + return NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, nil, nil, nil, nil, refs, locks, nodeUUID, sb.DataRegionOffset) } // createFile adds a regular file to the root directory as Create does. diff --git a/internal/goclusterfs/journal_crash_test.go b/internal/goclusterfs/journal_crash_test.go index caabca40..7d68a774 100644 --- a/internal/goclusterfs/journal_crash_test.go +++ b/internal/goclusterfs/journal_crash_test.go @@ -57,6 +57,12 @@ func crashAfterCommit(t *testing.T, fsys *FS, op string) <-chan struct{} { func remountAndReplay(t *testing.T, path string, offset, size uint64) *FS { t.Helper() fsys := openFormattedFS(t, path) + replayCrashedJournal(t, fsys, offset, size) + return fsys +} + +func replayCrashedJournal(t *testing.T, fsys *FS, offset, size uint64) { + t.Helper() region, err := journal.OpenRegionForRecovery(fsys.dev, offset, size) if err != nil { t.Fatalf("open the crashed node's journal: %v", err) @@ -77,7 +83,6 @@ func remountAndReplay(t *testing.T, path string, offset, size uint64) *FS { if err := fsys.lockMgr.ReleaseAdopted(adopted); err != nil { t.Fatalf("release the crashed node's locks: %v", err) } - return fsys } func expectCleanFsck(t *testing.T, fsys *FS) { diff --git a/internal/goclusterfs/power_cut_test.go b/internal/goclusterfs/power_cut_test.go new file mode 100644 index 00000000..81da88cb --- /dev/null +++ b/internal/goclusterfs/power_cut_test.go @@ -0,0 +1,397 @@ +package goclusterfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "math" + "os" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/snapshot" + "github.com/supporttools/GoClusterFS/internal/superblock" + "github.com/supporttools/GoClusterFS/internal/trim" +) + +var errPowerCut = errors.New("power cut") + +const maxRaceCuts = 48 + +type power struct { + limit int64 + writes atomic.Int64 +} + +func (p *power) cut() bool { return p.writes.Load() > p.limit } + +// cutPowerAfter lets the first n writes to fsys's device land and refuses the rest, as a power failure would. +func cutPowerAfter(fsys *FS, n int64) *power { + p := &power{limit: n} + fsys.dev.SetWriteGate(func(int64) error { + if p.writes.Add(1) > p.limit { + return errPowerCut + } + return nil + }) + return p +} + +type powerCutCase struct { + trim bool + // setup builds the state before the operation; all of its writes land. + setup func(t *testing.T, fsys *FS) + // op is the operation power fails during. It ignores errors, which the cut causes. + op func(fsys *FS, p *power) + // afterMount runs on the remounted node before fsck. + afterMount func(t *testing.T, fsys *FS) + // check runs after fsck found nothing but leaks and repair fixed them. + check func(t *testing.T, fsys *FS) +} + +// runPowerCuts runs the operation to completion once to count its writes, then once per write with power cut just +// before it. +func runPowerCuts(t *testing.T, c powerCutCase) { + total := runPowerCut(t, c, math.MaxInt64) + if total == 0 { + t.Fatal("the operation wrote nothing") + } + t.Logf("the operation makes %d writes", total) + step := int64(1) + // The race detector makes each cut many times slower, so it checks an even spread of them. + if raceDetector && total > maxRaceCuts { + step = (total + maxRaceCuts - 1) / maxRaceCuts + } + for n := int64(0); n < total; n += step { + t.Run(fmt.Sprintf("cut_after_%d_writes", n), func(t *testing.T) { runPowerCut(t, c, n) }) + } +} + +func runPowerCut(t *testing.T, c powerCutCase, n int64) int64 { + t.Helper() + fsys, path, offset, size := journaledFS(t) + if c.trim { + attachHolePunchingTrim(t, fsys, path) + } + c.setup(t, fsys) + p := cutPowerAfter(fsys, n) + c.op(fsys, p) + fsys.snapshotWkr.Stop() + writes := p.writes.Load() + fsys.dev.SetWriteGate(func(int64) error { return errPowerCut }) + + remounted := mountAfterPowerCut(t, path, offset, size, c.trim) + if c.afterMount != nil { + c.afterMount(t, remounted) + } + expectOnlyLeaks(t, remounted) + c.check(t, remounted) + return writes +} + +// mountAfterPowerCut opens the image as a new node in mount order: journal replay first, then snapshot recovery. +func mountAfterPowerCut(t *testing.T, path string, offset, size uint64, withTrim bool) *FS { + t.Helper() + fsys := openFormattedComponents(t, path) + if withTrim { + attachHolePunchingTrim(t, fsys, path) + } + replayCrashedJournal(t, fsys, offset, size) + startSnapshots(t, fsys) + deadline := time.Now().Add(10 * time.Second) + for { + pending := 0 + snaps, err := fsys.snapshotReg.List(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, s := range snaps { + if s.State != snapshot.StateActive { + pending++ + } + } + if pending == 0 { + return fsys + } + if time.Now().After(deadline) { + t.Fatalf("%d snapshots still being created or deleted 10s after mount", pending) + } + time.Sleep(10 * time.Millisecond) + } +} + +// attachHolePunchingTrim gives fsys a TRIM manager that flushes only when asked and discards by punching holes in the +// image, so a discard of a block in use shows as zeros. +func attachHolePunchingTrim(t *testing.T, fsys *FS, path string) { + t.Helper() + img, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := img.Close(); err != nil { + t.Error(err) + } + }) + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + const keepSize, punchHole = 0x01, 0x02 + tm := trim.NewManager(fsys.dev, trim.Config{ + BatchSize: 1 << 16, + MaxAge: time.Hour, + DataOffset: sb.DataRegionOffset, + Enabled: true, + Discard: func(off, length uint64) error { + return syscall.Fallocate(int(img.Fd()), keepSize|punchHole, int64(off), int64(length)) + }, + }) + fsys.bitmapMgr.SetTrimManager(tm) + fsys.trimMgr = tm +} + +// expectOnlyLeaks requires fsck to find nothing but leaked blocks or inodes, and repair to fix those. +func expectOnlyLeaks(t *testing.T, fsys *FS) { + t.Helper() + r := runFsck(t, fsys, fsck.DefaultOptions()) + if len(r.Problems) == 0 { + return + } + for _, p := range r.Problems { + if !isLeak(p.Message) { + t.Errorf("fsck after the power cut found more than leaks:%s", problemList(r)) + break + } + } + t.Logf("leaks to repair:%s", problemList(r)) + repair := fsck.DefaultOptions() + repair.Mode = fsck.Repair + runFsck(t, fsys, repair) + expectCleanFsck(t, fsys) +} + +func isLeak(msg string) bool { + for _, leak := range []string{"leaked", "will never be freed", "unnamed: no directory names it", "belongs to no snapshot"} { + if strings.Contains(msg, leak) { + return true + } + } + return false +} + +func readWhole(t *testing.T, f *File) []byte { + t.Helper() + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + t.Fatal(err) + } + res, errno := f.Read(context.Background(), nil, make([]byte, ino.Size), 0) + if errno != 0 { + t.Fatalf("inode %d: read: %v", f.inodeNum, errno) + } + got, _ := res.Bytes(make([]byte, ino.Size)) + return got +} + +// expectEachBlock requires every block of f to hold either its old or its new data. +func expectEachBlock(t *testing.T, f *File, old, updated []byte) { + t.Helper() + got := readWhole(t, f) + if len(got) != len(old) { + t.Fatalf("inode %d: %d bytes, want %d", f.inodeNum, len(got), len(old)) + } + for i := 0; i < len(old); i += BlockSize { + end := min(i+BlockSize, len(old)) + if !bytes.Equal(got[i:end], old[i:end]) && !bytes.Equal(got[i:end], updated[i:end]) { + t.Fatalf("inode %d: block %d holds neither its old nor its new data", f.inodeNum, i/BlockSize) + } + } +} + +// expectSnapshot requires the named snapshot to be active with want as its data, and reports whether it exists. +func expectSnapshot(t *testing.T, fsys *FS, name string, want []byte) bool { + t.Helper() + desc, err := fsys.snapshotReg.GetByName(context.Background(), name) + if errors.Is(err, snapshot.ErrSnapshotNotFound) { + return false + } + if err != nil { + t.Fatal(err) + } + if desc.State != snapshot.StateActive { + t.Fatalf("snapshot %s is %s after recovery", name, desc.State) + } + if !bytes.Equal(readWhole(t, &File{fs: fsys, inodeNum: desc.DataInode}), want) { + t.Fatalf("snapshot %s changed", name) + } + return true +} + +func lookupMaybe(t *testing.T, fsys *FS, name string) *File { + t.Helper() + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + num, _, err := fsys.dirMgr.Lookup(context.Background(), root, name) + if err != nil { + return nil + } + return &File{fs: fsys, inodeNum: uint32(num)} +} + +// waitForSnapshotDelete waits until the delete job freed the snapshot's data inode or power failed. +func waitForSnapshotDelete(fsys *FS, p *power, dataInode uint32) { + deadline := time.Now().Add(10 * time.Second) + for !p.cut() && time.Now().Before(deadline) { + if ino, err := fsys.inodeTbl.Read(dataInode); err == nil && ino.Magic == 0 { + return + } + time.Sleep(time.Millisecond) + } +} + +// Power fails during a write that copies two blocks a snapshot shares. Each block keeps its old or its new data, and +// the snapshot keeps the data it captured. +func TestPowerCutDuringCopyOnWrite(t *testing.T) { + old := pattern(1, 3*BlockSize) + update := pattern(9, BlockSize) + updated := overwritten(old, BlockSize+4096, update) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + f = createFile(t, fsys, "f") + writeAt(t, f, old, 0) + setxattr(t, f, "user.snapshot", "s1") + }, + op: func(fsys *FS, _ *power) { + _, _ = f.Write(context.Background(), nil, update, BlockSize+4096) + }, + check: func(t *testing.T, fsys *FS) { + expectEachBlock(t, lookupFile(t, fsys, "f"), old, updated) + if !expectSnapshot(t, fsys, "s1", old) { + t.Fatal("snapshot s1 is gone") + } + }, + }) +} + +// Power fails while a snapshot is created. The file and the older snapshot keep their data, and the new snapshot is +// either gone or complete. +func TestPowerCutDuringSnapshotCreate(t *testing.T) { + old := pattern(1, 3*BlockSize) + current := overwritten(old, 0, pattern(5, BlockSize)) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + f = createFile(t, fsys, "f") + writeAt(t, f, old, 0) + setxattr(t, f, "user.snapshot", "s0") + writeAt(t, f, current[:BlockSize], 0) + }, + op: func(fsys *FS, _ *power) { + _ = f.Setxattr(context.Background(), "user.snapshot", []byte("s1"), 0) + }, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "f"), current) + if !expectSnapshot(t, fsys, "s0", old) { + t.Fatal("snapshot s0 is gone") + } + expectSnapshot(t, fsys, "s1", current) + }, + }) +} + +// Power fails while a snapshot is deleted, including its background job. The file and the other snapshot keep their +// data, and the deleted snapshot is either gone or intact. +func TestPowerCutDuringSnapshotDelete(t *testing.T) { + old := pattern(1, 3*BlockSize) + mid := overwritten(old, BlockSize, pattern(5, BlockSize)) + current := overwritten(mid, 2*BlockSize, pattern(7, BlockSize)) + var f *File + var s1Data uint32 + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + f = createFile(t, fsys, "f") + writeAt(t, f, old, 0) + setxattr(t, f, "user.snapshot", "s0") + writeAt(t, f, mid[BlockSize:2*BlockSize], BlockSize) + setxattr(t, f, "user.snapshot", "s1") + writeAt(t, f, current[2*BlockSize:], 2*BlockSize) + desc, err := fsys.snapshotReg.GetByName(context.Background(), "s1") + if err != nil { + t.Fatal(err) + } + s1Data = desc.DataInode + }, + op: func(fsys *FS, p *power) { + if errno := f.Removexattr(context.Background(), "user.snapshot.s1"); errno == 0 { + waitForSnapshotDelete(fsys, p, s1Data) + } + }, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "f"), current) + if !expectSnapshot(t, fsys, "s0", old) { + t.Fatal("snapshot s0 is gone") + } + expectSnapshot(t, fsys, "s1", mid) + }, + }) +} + +// Power fails while an unlink frees blocks that TRIM has queued. After replay a new file reuses the freed blocks and +// TRIM flushes: no discard may reach a block in use. +func TestPowerCutDuringUnlinkThenTrimAfterReuse(t *testing.T) { + kept := pattern(2, 2*BlockSize) + doomed := pattern(4, 3*BlockSize) + fresh := pattern(6, 3*BlockSize) + runPowerCuts(t, powerCutCase{ + trim: true, + setup: func(t *testing.T, fsys *FS) { + writeAt(t, createFile(t, fsys, "keep"), kept, 0) + writeAt(t, createFile(t, fsys, "old"), doomed, 0) + }, + op: func(fsys *FS, _ *power) { + _ = fsys.Unlink(context.Background(), "old") + }, + afterMount: func(t *testing.T, fsys *FS) { + writeAt(t, createFile(t, fsys, "new"), fresh, 0) + fsys.trimMgr.Flush() + }, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "keep"), kept) + checkFile(t, lookupFile(t, fsys, "new"), fresh) + if old := lookupMaybe(t, fsys, "old"); old != nil { + checkFile(t, old, doomed) + } + }, + }) +} + +// Power fails while a file is cloned. The file keeps its data, and the clone is either gone or a full copy. +func TestPowerCutDuringClone(t *testing.T) { + data := pattern(3, 3*BlockSize) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + f = createFile(t, fsys, "f") + writeAt(t, f, data, 0) + }, + op: func(fsys *FS, _ *power) { + _ = f.Setxattr(context.Background(), "user.clone", []byte("/c"), 0) + }, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "f"), data) + if c := lookupMaybe(t, fsys, "c"); c != nil { + checkFile(t, c, data) + } + }, + }) +} diff --git a/internal/goclusterfs/race_off_test.go b/internal/goclusterfs/race_off_test.go new file mode 100644 index 00000000..8516ed72 --- /dev/null +++ b/internal/goclusterfs/race_off_test.go @@ -0,0 +1,5 @@ +//go:build !race + +package goclusterfs + +const raceDetector = false diff --git a/internal/goclusterfs/race_on_test.go b/internal/goclusterfs/race_on_test.go new file mode 100644 index 00000000..ac0ce6fe --- /dev/null +++ b/internal/goclusterfs/race_on_test.go @@ -0,0 +1,5 @@ +//go:build race + +package goclusterfs + +const raceDetector = true diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index f9e3c894..e3d0dfd8 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -376,15 +376,13 @@ func TestSnapshotRecoveryAfterCrash(t *testing.T) { blocks := []uint64{physicalBlock(t, fsys, f.inodeNum, 0), physicalBlock(t, fsys, f.inodeNum, 1)} // Crash after the blocks were shared, before the snapshot became active. - data, err := fsys.claimInode(ctx) + data, err := fsys.claimInode(ctx, func(ino *inode.Inode) { + ino.Mode = inode.S_IFREG | 0o644 + ino.Flags |= inode.FlagSnapshotData + }) if err != nil { t.Fatal(err) } - data.Mode = inode.S_IFREG | 0o644 - data.Flags |= inode.FlagSnapshotData - if err := fsys.inodeTbl.Write(ctx, data); err != nil { - t.Fatal(err) - } if _, err := fsys.snapshotReg.Create(ctx, "half-created", f.inodeNum, 0, data.InodeNum, 2, uint64(len(content)), snapshot.FlagReadOnly); err != nil { t.Fatal(err) } diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 7f3586bf..4be9a261 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -434,6 +434,16 @@ func (t *Table) ClaimSlot(ctx context.Context, num uint32) (*Inode, error) { // ClaimSlotIf claims the slot if it is free and ok accepts the free inode, read under the slot's lock. Otherwise it // returns ErrInodeInUse. func (t *Table) ClaimSlotIf(ctx context.Context, num uint32, ok func(free *Inode) bool) (*Inode, error) { + return t.claimSlot(ctx, num, ok, nil) +} + +// ClaimSlotAs claims a free slot and lets init fill in the new inode before its first write, so a crash cannot leave +// the slot claimed without those fields. +func (t *Table) ClaimSlotAs(ctx context.Context, num uint32, init func(*Inode)) (*Inode, error) { + return t.claimSlot(ctx, num, func(*Inode) bool { return true }, init) +} + +func (t *Table) claimSlot(ctx context.Context, num uint32, ok func(free *Inode) bool, init func(*Inode)) (*Inode, error) { if uint64(num) >= t.totalInodes { return nil, ErrInodeOutOfRange } @@ -475,6 +485,9 @@ func (t *Table) ClaimSlotIf(ctx context.Context, num uint32, ok func(free *Inode } newInode := t.newInodeInSlot(num, existing) + if init != nil { + init(newInode) + } if err := t.writeInodeUnlocked(newInode); err != nil { return nil, fmt.Errorf("failed to write new inode: %w", err) } diff --git a/internal/journal/torn_tail_test.go b/internal/journal/torn_tail_test.go new file mode 100644 index 00000000..5fd1f79d --- /dev/null +++ b/internal/journal/torn_tail_test.go @@ -0,0 +1,45 @@ +package journal + +import ( + "slices" + "testing" + + "github.com/supporttools/GoClusterFS/internal/storage" +) + +func zeroJournalSector(t *testing.T, j *Journal, pos uint32) { + t.Helper() + buf := storage.GetBuffer(storage.SectorSize) + defer storage.PutBuffer(buf) + clear(buf.Bytes()) + if _, err := j.region.dev.WriteAt(buf.Bytes(), j.region.dataOffset(pos)); err != nil { + t.Fatal(err) + } +} + +// Power fails while the last transaction's sectors are written, and its last sector never lands. Recovery replays the +// transactions before it and stops there. +func TestRecoverStopsAtTornLastTransaction(t *testing.T) { + j := newTestJournal(t) + seqs := commitIntents(t, j, false, false) + + txn := j.Begin() + for i := range 200 { + if err := txn.LogInodeCreate(testInodePayload(uint64(1000 + i))); err != nil { + t.Fatal(err) + } + } + start := j.region.TailOffset() + if _, err := j.Commit(txn); err != nil { + t.Fatal(err) + } + end := j.region.TailOffset() + if end-start < 2*storage.SectorSize { + t.Fatalf("the transaction takes %d bytes, want several sectors", end-start) + } + zeroJournalSector(t, j, end-storage.SectorSize) + + if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs) { + t.Errorf("replayed %v, want %v", got, seqs) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 14618596..66abdf66 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1417,6 +1417,54 @@ scenarios: environments: vm: run-vm-tests.sh::test_fsck_repair_after_power_failure + - id: cow-write-power-failure + name: Power failure during copy-on-write writes to a snapshotted file + description: Power fails at each device write of a write that copies blocks a snapshot shares; after replay fsck finds only leaks, which repair fixes, each block holds its old or new data and the snapshot is unchanged. The VM test rewrites blocks with fsync until a power failure and checks every acknowledged write + priority: high + environments: + go: internal/goclusterfs/power_cut_test.go::TestPowerCutDuringCopyOnWrite + docker: TODO # a container stop flushes the page cache and cannot cut power + vm: run-vm-tests.sh::test_cow_write_power_failure + + - id: snapshot-create-delete-power-failure + name: Power failure while snapshots are created and deleted + description: Power fails at each device write of a snapshot create, and of a delete with its background job; the file and other snapshots keep their data and the snapshot is either gone or complete. The VM test creates and deletes snapshots until a power failure and checks each acknowledged step + priority: high + environments: + go: internal/goclusterfs/power_cut_test.go::TestPowerCutDuringSnapshotCreate + docker: TODO # a container stop flushes the page cache and cannot cut power + vm: run-vm-tests.sh::test_snapshot_create_delete_power_failure + + - id: snapshot-delete-power-failure-go + name: Power failure at each write of a snapshot delete + description: Power fails at each device write of a snapshot delete and its background job; the file and the other snapshot keep their data and the deleted snapshot is either gone or intact + priority: go_only + environments: + go: internal/goclusterfs/power_cut_test.go::TestPowerCutDuringSnapshotDelete + + - id: clone-power-failure-go + name: Power failure at each write of a clone + description: Power fails at each device write of a clone; the source keeps its data and the clone is either gone or a full copy, and fsck finds only leaks + priority: go_only + environments: + go: internal/goclusterfs/power_cut_test.go::TestPowerCutDuringClone + + - id: trim-reuse-power-failure + name: Power failure between freeing blocks and discarding them + description: Power fails at each device write of an unlink whose blocks TRIM has queued; after replay a new file reuses the blocks and TRIM flushes with discards that punch holes, and no file loses data. The VM test, with discard passed to the shared disk, removes and rewrites files until a power failure and checks every acknowledged file + priority: high + environments: + go: internal/goclusterfs/power_cut_test.go::TestPowerCutDuringUnlinkThenTrimAfterReuse + docker: TODO # a container stop flushes the page cache and cannot cut power + vm: run-vm-tests.sh::test_trim_reuse_power_failure + + - id: journal-torn-tail + name: Recovery stops at a torn last transaction + description: The last sector of a multi-sector transaction never lands; recovery replays the transactions before it and stops + priority: go_only + environments: + go: internal/journal/torn_tail_test.go::TestRecoverStopsAtTornLastTransaction + - id: journal-stale-record-after-tail name: Recovery ignores a stale record where a lost one should be description: The superblock covering a new record lands but the record does not, leaving a record from the previous mount or the previous lap in its place; recovery must not replay it diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index c36d6889..00f904ad 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -3201,6 +3201,276 @@ test_fsck_repair_after_power_failure() { return 0 } +# ============================================================================= +# Power failures during copy-on-write, snapshots and TRIM +# ============================================================================= +# A workload script runs on the victim and prints a line after each step it +# finished with fsync. The victim loses power mid-workload; every step it +# acknowledged must survive replay, and fsck may find nothing but leaks. + +# Prints the letter round $1 of a workload writes. +round_letter() { + printf "\\$(printf %03o $((65 + $1 % 26)))" +} + +install_power_failure_scripts() { + local vm_id=$1 + ssh_run "$vm_id" "cat > /tmp/pf_letters.sh && chmod +x /tmp/pf_letters.sh" <<'EOF' +#!/bin/sh +# Prints, for each 64KB chunk of $1 from byte $2 for $3 chunks, its letter, or the distinct bytes of a mixed chunk +# in parentheses. +for L in a A B C D E F G H I J K L M N O P Q R S T U V W X Y Z; do + echo "$(head -c 65536 /dev/zero | tr '\0' "$L" | md5sum | cut -c1-32) $L" +done > /tmp/pf_letter_md5s +k=0 +while [ "$k" -lt "$3" ]; do + chunk="dd if=$1 bs=65536 skip=$(($2 / 65536 + k)) count=1" + m=$($chunk 2>/dev/null | md5sum | cut -c1-32) + L=$(grep "^$m " /tmp/pf_letter_md5s | cut -c34) + if [ -z "$L" ]; then + L="($($chunk 2>/dev/null | od -An -c -v | tr -s ' ' '\n' | grep -v '^$' | sort -u | tr -d '\n'))" + fi + printf %s "$L" + k=$((k + 1)) +done +echo +EOF + ssh_run "$vm_id" "cat > /tmp/pf_cow.sh && chmod +x /tmp/pf_cow.sh" <<'EOF' +#!/bin/sh +# Rewrites block (round % 4) of $1 with the round's letter, with fsync, and acknowledges each round. +i=0 +while :; do + i=$((i + 1)) + c=$(printf "\\$(printf %03o $((65 + i % 26)))") + head -c 1048576 /dev/zero | tr '\0' "$c" | dd of="$1" bs=1M seek=$((i % 4)) conv=notrunc,fsync 2>/dev/null || exit 1 + echo "ack $i" +done +EOF + ssh_run "$vm_id" "cat > /tmp/pf_snap.sh && chmod +x /tmp/pf_snap.sh" <<'EOF' +#!/bin/sh +# Creates snapshot pf_s of $1 and deletes the one from two rounds back, acknowledging each step. +i=0 +while :; do + i=$((i + 1)) + setfattr -n user.snapshot -v "pf_s$i" "$1" || exit 1 + echo "created $i" + if [ "$i" -gt 2 ]; then + setfattr -x "user.snapshot.pf_s$((i - 2))" "$1" || exit 1 + echo "deleted $((i - 2))" + fi +done +EOF + ssh_run "$vm_id" "cat > /tmp/pf_trim.sh && chmod +x /tmp/pf_trim.sh" <<'EOF' +#!/bin/sh +# Writes a 2MB file of the round's letter in $1 with fsync after removing the file from three rounds back, so new +# files reuse blocks TRIM has queued, and acknowledges each round. +i=0 +while :; do + i=$((i + 1)) + rm -f "$1/trim_pf_$((i - 3)).bin" + c=$(printf "\\$(printf %03o $((65 + i % 26)))") + head -c 2097152 /dev/zero | tr '\0' "$c" | dd of="$1/trim_pf_$i.bin" bs=1M conv=fsync 2>/dev/null || exit 1 + echo "ack $i" +done +EOF +} + +# Cuts power to VM $1 while it runs workload $2 for $3 seconds, writing its acknowledgements to file $4. +power_fail_during() { + local victim=$1 workload=$2 seconds=$3 acks=$4 + ssh_run "$victim" "$workload" > "$acks" 2>/dev/null & + local runner=$! + sleep "$seconds" + log_info "Simulating power failure on VM $victim after $(wc -l < "$acks") acknowledged steps..." + kill_vm "$victim" + wait "$runner" 2>/dev/null || true +} + +# Restarts VM $1 with journal replay, then requires fsck to find nothing but leaks and -repair to fix them. +restart_and_expect_only_leaks() { + local victim=$1 fsck=/usr/local/bin/goclusterfs-fsck vm_id out problems + start_vm "$victim" false + wait_vm_healthy "$victim" || return 1 + setup_goclusterfs "$victim" false || return 1 + sleep 3 # snapshot recovery jobs finish in the background after mount + for vm_id in $(seq 1 "$NODE_COUNT"); do + if ! unmount_goclusterfs "$vm_id"; then + log_error "VM $vm_id: GoClusterFS did not unmount" + return 1 + fi + done + + out=$(ssh_run 1 "$fsck /dev/vda 2>&1") + problems=$(echo "$out" | grep -E '^ *\[(WARN|ERROR)\]') + if echo "$problems" | grep -v -e '^$' -e 'leaked' -e 'will never be freed' -e 'unnamed: no directory names it' -e 'belongs to no snapshot' | grep -q .; then + log_error "fsck after the power failure found more than leaks:" + echo "$problems" | head -20 >&2 + return 1 + fi + if [[ -n "$problems" ]]; then + log_info "fsck found $(echo "$problems" | wc -l) leaks; repairing" + ssh_run 1 "$fsck -repair /dev/vda > /dev/null 2>&1" + out=$(ssh_run 1 "$fsck /dev/vda 2>&1") + if echo "$out" | grep -qE '^ *\[(WARN|ERROR)\]'; then + log_error "fsck after repair still finds problems:" + echo "$out" | grep -E '^ *\[(WARN|ERROR)\]' | head -20 >&2 + return 1 + fi + fi + for vm_id in $(seq 1 "$NODE_COUNT"); do + setup_goclusterfs "$vm_id" false || return 1 + done +} + +test_cow_write_power_failure() { + log_info "Testing a power failure during copy-on-write writes to a snapshotted file..." + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping: Need at least 2 VMs for this test" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + local victim=2 f=${GOCLUSTERFS_MOUNT}/cow_pf.bin acks + acks=$(mktemp) + install_power_failure_scripts "$victim" + install_power_failure_scripts 1 + ssh_run "$victim" "head -c 4194304 /dev/zero | tr '\\0' a > $f && sync && setfattr -n user.snapshot -v cow_pf_snap $f" || return 1 + + power_fail_during "$victim" "/tmp/pf_cow.sh $f" 6 "$acks" + local last + last=$(awk '/^ack/ {n = $2} END {print n + 0}' "$acks") + restart_and_expect_only_leaks "$victim" || return 1 + + local b r letters want maybe + for b in 0 1 2 3; do + want=a + for ((r = last; r > 0; r--)); do + if (( r % 4 == b )); then want=$(round_letter "$r"); break; fi + done + maybe=$want + for r in $((last + 1)) $((last + 2)); do + (( r % 4 == b )) && maybe="$maybe$(round_letter "$r")" + done + letters=$(ssh_run 1 "/tmp/pf_letters.sh $f $((b * 1048576)) 16") + # A write in flight may stop partway through a chunk, but only old and new bytes may show. + if [[ ! "$letters" =~ ^([$maybe]|\([$maybe]+\)){16}$ ]]; then + log_error "block $b after $last acknowledged writes holds '$letters', want only '$maybe'" + return 1 + fi + done + ssh_run 1 "setfattr -n user.snapshot.clone -v cow_pf_snap:/cow_pf_snap_check.bin $f" || return 1 + letters=$(ssh_run 1 "/tmp/pf_letters.sh ${GOCLUSTERFS_MOUNT}/cow_pf_snap_check.bin 0 64") + if [[ "$letters" != "$(printf 'a%.0s' $(seq 64))" ]]; then + log_error "snapshot cow_pf_snap changed: '$letters'" + return 1 + fi + log_info "$last acknowledged copy-on-write writes and the snapshot survived the power failure" + rm -f "$acks" +} + +test_snapshot_create_delete_power_failure() { + log_info "Testing a power failure while snapshots are created and deleted..." + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping: Need at least 2 VMs for this test" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + local victim=2 f=${GOCLUSTERFS_MOUNT}/snap_pf.bin acks + acks=$(mktemp) + install_power_failure_scripts "$victim" + ssh_run "$victim" "dd if=/dev/urandom of=$f bs=1M count=3 2>/dev/null && sync" || return 1 + local base + base=$(ssh_run "$victim" "md5sum $f | cut -d' ' -f1") + + power_fail_during "$victim" "/tmp/pf_snap.sh $f" 6 "$acks" + local created deleted + created=$(awk '/^created/ {n = $2} END {print n + 0}' "$acks") + deleted=$(awk '/^deleted/ {n = $2} END {print n + 0}' "$acks") + restart_and_expect_only_leaks "$victim" || return 1 + + if [[ "$(ssh_run 1 "md5sum $f | cut -d' ' -f1")" != "$base" ]]; then + log_error "snap_pf.bin changed" + return 1 + fi + # Recovery finishes snapshots left creating or deleting in the background after mount. Entries are name:state. + local listed waited=0 + while listed=$(ssh_run 1 "getfattr --only-values -n user.snapshots $f 2>/dev/null" | tr ',' '\n'); \ + [[ $waited -lt 20 ]] && echo "$listed" | awk -F: -v d="$deleted" \ + '$1 ~ /^pf_s/ && ($2 != "active" || substr($1, 5) + 0 <= d) {found = 1} END {exit !found}'; do + sleep 1 + waited=$((waited + 1)) + done + listed=$(echo "$listed" | cut -d: -f1) + local i name + for ((i = 1; i <= created + 1; i++)); do + name=pf_s$i + if ! echo "$listed" | tr ',' '\n' | grep -qx "$name"; then + if (( i <= created && i > deleted + 1 )); then + log_error "snapshot $name was acknowledged and not deleted, but is gone (listed: $listed)" + return 1 + fi + continue + fi + if (( i <= deleted )); then + log_error "snapshot $name was acknowledged as deleted, but is still listed" + return 1 + fi + ssh_run 1 "setfattr -n user.snapshot.clone -v $name:/${name}_check.bin $f" || return 1 + if [[ "$(ssh_run 1 "md5sum ${GOCLUSTERFS_MOUNT}/${name}_check.bin | cut -d' ' -f1")" != "$base" ]]; then + log_error "snapshot $name changed" + return 1 + fi + done + log_info "$created snapshots created and $deleted deleted before the power failure; the survivors are intact" + rm -f "$acks" +} + +test_trim_reuse_power_failure() { + log_info "Testing a power failure while freed blocks wait for TRIM and new files reuse them..." + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping: Need at least 2 VMs for this test" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + local victim=2 acks + acks=$(mktemp) + install_power_failure_scripts "$victim" + install_power_failure_scripts 1 + ssh_run "$victim" "dd if=/dev/urandom of=${GOCLUSTERFS_MOUNT}/trim_keep.bin bs=1M count=4 2>/dev/null && sync" || return 1 + local keep + keep=$(ssh_run "$victim" "md5sum ${GOCLUSTERFS_MOUNT}/trim_keep.bin | cut -d' ' -f1") + + # TRIM flushes freed blocks after 5s, so the workload runs through several flushes. + power_fail_during "$victim" "/tmp/pf_trim.sh ${GOCLUSTERFS_MOUNT}" 12 "$acks" + local last + last=$(awk '/^ack/ {n = $2} END {print n + 0}' "$acks") + restart_and_expect_only_leaks "$victim" || return 1 + + if [[ "$(ssh_run 1 "md5sum ${GOCLUSTERFS_MOUNT}/trim_keep.bin | cut -d' ' -f1")" != "$keep" ]]; then + log_error "trim_keep.bin changed" + return 1 + fi + local i letters + for ((i = 1; i <= last; i++)); do + letters=$(ssh_run 1 "test -e ${GOCLUSTERFS_MOUNT}/trim_pf_$i.bin && /tmp/pf_letters.sh ${GOCLUSTERFS_MOUNT}/trim_pf_$i.bin 0 32") + if [[ -z "$letters" ]]; then + if (( i == last )); then + log_error "trim_pf_$last.bin was acknowledged and never removed, but is gone" + return 1 + fi + continue + fi + if [[ "$letters" != "$(printf "$(round_letter "$i")%.0s" $(seq 32))" ]]; then + log_error "trim_pf_$i.bin was acknowledged but holds '$letters'" + return 1 + fi + done + if ! grep -q "\[trim\] .*flushing" "$(get_vm_mount_log "$victim")" "$(get_vm_mount_log 1)"; then + log_warn "no TRIM flush ran during the test" + fi + log_info "$last acknowledged files survived the power failure and every TRIM flush" + rm -f "$acks" +} + # ============================================================================= # Main Test Execution # ============================================================================= @@ -3671,6 +3941,9 @@ main() { run_test "Reversed Write Order" test_reversed_write_order run_test "Concurrent Writes Power Failure" test_concurrent_writes_power_failure run_test "fsck Repair After Power Failure" test_fsck_repair_after_power_failure + run_test "Copy-on-Write Power Failure" test_cow_write_power_failure + run_test "Snapshot Create/Delete Power Failure" test_snapshot_create_delete_power_failure + run_test "TRIM Reuse Power Failure" test_trim_reuse_power_failure run_test "Stalled Node Writes Stop At Write Gate" test_stalled_node_writes_stop_at_write_gate # Summary diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 09d7a837..43c704f5 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -398,7 +398,7 @@ boot_vm() { -smp ${VM_CPUS} \ -cdrom "${ISO_PATH}" \ -boot d \ - -drive "file=${SHARED_DISK_PATH},format=raw,if=virtio,cache=none,file.locking=off" \ + -drive "file=${SHARED_DISK_PATH},format=raw,if=virtio,cache=none,discard=unmap,file.locking=off" \ -drive "file=${VM_RUN_DIR}/setup.img,format=raw,if=virtio,readonly=on" \ -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ -device "virtio-net-pci,netdev=net0" \