From 2762ea9ceaa80459f5fe49a2e7c6950c07ad6e80 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 05:07:29 -0600 Subject: [PATCH 1/2] test(fsck): free the blocks an interrupted truncate leaves A node crashes after a truncate writes the shorter inode but before it frees the cut blocks. Since #65, replay frees nothing for that record, so the blocks stay allocated. The test stalls the truncate's free on a refcount lock another node holds, crashes by closing the device, replays the journal, and checks that fsck -repair frees exactly the 2 cut blocks and that both files keep their data. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fsck_reclaim_test.go | 67 +++++++++++++++++++++++ tests/test-manifest.yaml | 7 +++ 2 files changed, 74 insertions(+) diff --git a/internal/goclusterfs/fsck_reclaim_test.go b/internal/goclusterfs/fsck_reclaim_test.go index e484aaec..ee87206b 100644 --- a/internal/goclusterfs/fsck_reclaim_test.go +++ b/internal/goclusterfs/fsck_reclaim_test.go @@ -422,3 +422,70 @@ func TestFsckRepairFreesEmptySFBContainers(t *testing.T) { checkFile(t, lookupFile(t, fsys, "small.txt"), pattern(1, 5000)) checkFile(t, lookupFile(t, fsys, "moved.txt"), append(pattern(2, 10000), pattern(3, 10000)...)) } + +// holdRefcountLock makes another node hold the refcount lock until the test +// ends, so frees on fsys wait for it. +func holdRefcountLock(t *testing.T, fsys *FS) { + t.Helper() + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + slot := int(snapFSNodes.Add(1)-1) % 64 + var otherUUID [16]byte + copy(otherUUID[:], "lockholder123456") + otherUUID[15] = byte(slot) + other := lock.NewManager(fsys.dev, sb.LockBitmapOffset, sb.LockBitmapSize, otherUUID, slot) + if err := other.Acquire(context.Background(), lock.LockTypeRefCount, 0, lock.LockModeExclusive); err != nil { + t.Fatal(err) + } +} + +// A crash after a truncate wrote the shorter inode, but before it freed the +// blocks it cut, leaves them allocated: replay frees nothing for an +// interrupted truncate. Repair frees them. +func TestFsckRepairReclaimsInterruptedTruncate(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + victim := createFile(t, fsys, "victim.bin") + writeAt(t, victim, pattern(1, 3*BlockSize), 0) + other := createFile(t, fsys, "other.bin") + writeAt(t, other, pattern(2, 2*BlockSize), 0) + free := fsys.bitmapMgr.FreeCount() + + // The truncate's free waits for the lock; the node crashes while it waits. + holdRefcountLock(t, fsys) + go tryTruncate(victim, BlockSize) + deadline := time.Now().Add(10 * time.Second) + for { + ino, err := fsys.inodeTbl.Read(victim.inodeNum) + if err == nil && ino.Size == BlockSize { + break + } + if time.Now().After(deadline) { + t.Fatalf("truncate did not write the inode: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + fsys.dev.Close() + + remounted := remountAndReplay(t, path, offset, size) + if got := remounted.bitmapMgr.FreeCount(); got != free { + t.Fatalf("%d free blocks after replay, want the 2 cut blocks still allocated (%d)", got, free) + } + if r := runFsck(t, remounted, fsck.DefaultOptions()); !strings.Contains(problemList(r), "leaked") { + t.Fatalf("check did not report the cut blocks:%s", problemList(r)) + } + repair := fsck.DefaultOptions() + repair.Mode = fsck.Repair + if r := runFsck(t, remounted, repair); r.ExitCode() != fsck.ExitCorrected { + t.Fatalf("repair exit %d, want %d:%s", r.ExitCode(), fsck.ExitCorrected, problemList(r)) + } + expectCleanFsck(t, remounted) + + remounted = openFormattedFS(t, path) + if got := remounted.bitmapMgr.FreeCount(); got != free+2 { + t.Fatalf("%d free blocks after repair, want %d", got, free+2) + } + checkFile(t, lookupFile(t, remounted, "victim.bin"), pattern(1, BlockSize)) + checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, 2*BlockSize)) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 66b65ad6..0ad29aaa 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -963,6 +963,13 @@ scenarios: environments: go: internal/goclusterfs/fsck_reclaim_test.go::TestFsckRepairReclaimsLeakedBlocks + - id: fsck-reclaim-interrupted-truncate + name: fsck -repair frees blocks an interrupted truncate leaves + description: A node crashes after a truncate writes the shorter inode but before it frees the cut blocks; replay frees nothing, check reports the blocks as leaked, repair frees them, and both files keep their data + priority: go_only + environments: + go: internal/goclusterfs/fsck_reclaim_test.go::TestFsckRepairReclaimsInterruptedTruncate + - id: fsck-reclaim-empty-sfb-containers name: fsck -repair frees SFB containers no file uses description: A file that grows out of its SFB slot leaves its container empty; check reports it as leaked, repair frees it and keeps a container another file still uses, and both files keep their data From 681b7844c2094590e0dd0ccca4fff72a7b41e2a4 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 05:19:41 -0600 Subject: [PATCH 2/2] test(fsck): cover an interrupted subdirectory unlink and either replay behavior The truncate test required replay to leave the cut blocks allocated. A shared helper now accepts whatever of the released blocks replay leaves: check must name exactly those, repair must free exactly those, and the allocated set must end as before minus the released blocks. The test keeps passing once replay frees these blocks itself. The new test crashes while unlinking a file in a subdirectory, after its entry is gone and before its blocks are freed. Replay deletes the inode and frees nothing for the BlockFree record, so the blocks leak until repair frees them. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fsck_reclaim_test.go | 183 +++++++++++++++++++--- tests/test-manifest.yaml | 9 +- 2 files changed, 165 insertions(+), 27 deletions(-) diff --git a/internal/goclusterfs/fsck_reclaim_test.go b/internal/goclusterfs/fsck_reclaim_test.go index ee87206b..1338b5bf 100644 --- a/internal/goclusterfs/fsck_reclaim_test.go +++ b/internal/goclusterfs/fsck_reclaim_test.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "errors" "fmt" + "maps" + "slices" "strings" "syscall" "testing" @@ -441,51 +443,180 @@ func holdRefcountLock(t *testing.T, fsys *FS) { } } -// A crash after a truncate wrote the shorter inode, but before it freed the -// blocks it cut, leaves them allocated: replay frees nothing for an -// interrupted truncate. Repair frees them. -func TestFsckRepairReclaimsInterruptedTruncate(t *testing.T) { - fsys, path, offset, size := journaledFS(t) - victim := createFile(t, fsys, "victim.bin") - writeAt(t, victim, pattern(1, 3*BlockSize), 0) - other := createFile(t, fsys, "other.bin") - writeAt(t, other, pattern(2, 2*BlockSize), 0) - free := fsys.bitmapMgr.FreeCount() +func allocatedBlocks(t *testing.T, fsys *FS) map[uint64]bool { + t.Helper() + out := map[uint64]bool{} + for b := uint64(0); b < fsys.bitmapMgr.TotalBlocks(); b++ { + allocated, err := fsys.bitmapMgr.IsAllocated(b) + if err != nil { + t.Fatal(err) + } + if allocated { + out[b] = true + } + } + return out +} - // The truncate's free waits for the lock; the node crashes while it waits. +// crashWhileFreeing runs op until its block free waits on a refcount lock another +// node holds and stuck reports its earlier changes on disk, then crashes the +// node there by closing its device. +func crashWhileFreeing(t *testing.T, fsys *FS, op func(), stuck func() bool) { + t.Helper() holdRefcountLock(t, fsys) - go tryTruncate(victim, BlockSize) + go op() deadline := time.Now().Add(10 * time.Second) - for { - ino, err := fsys.inodeTbl.Read(victim.inodeNum) - if err == nil && ino.Size == BlockSize { - break - } + for !stuck() { if time.Now().After(deadline) { - t.Fatalf("truncate did not write the inode: %v", err) + t.Fatal("the operation did not reach its block free") } time.Sleep(10 * time.Millisecond) } fsys.dev.Close() +} +// replayAndRepair remounts after the crash, replays the journal, and checks that +// fsck -repair leaves exactly before minus released allocated. Replay may leave +// any of released allocated; check must report those as leaked and repair must +// free them and nothing else. It returns the image mounted again. +func replayAndRepair(t *testing.T, path string, offset, size uint64, before map[uint64]bool, released []uint64) *FS { + t.Helper() + want := maps.Clone(before) + for _, b := range released { + delete(want, b) + } remounted := remountAndReplay(t, path, offset, size) - if got := remounted.bitmapMgr.FreeCount(); got != free { - t.Fatalf("%d free blocks after replay, want the 2 cut blocks still allocated (%d)", got, free) + var leaked []uint64 + for b := range allocatedBlocks(t, remounted) { + if !want[b] { + leaked = append(leaked, b) + } + } + slices.Sort(leaked) + for _, b := range leaked { + if !slices.Contains(released, b) { + t.Fatalf("block %d is allocated after replay but the operation did not release it", b) + } + } + for b := range want { + if allocated, _ := remounted.bitmapMgr.IsAllocated(b); !allocated { + t.Fatalf("replay freed block %d, which should stay allocated", b) + } } - if r := runFsck(t, remounted, fsck.DefaultOptions()); !strings.Contains(problemList(r), "leaked") { - t.Fatalf("check did not report the cut blocks:%s", problemList(r)) + + check := runFsck(t, remounted, fsck.DefaultOptions()) + for _, b := range leaked { + if !strings.Contains(problemList(check), fmt.Sprintf("block %d: leaked", b)) { + t.Fatalf("check did not report block %d, which replay left allocated:%s", b, problemList(check)) + } + } + if len(leaked) == 0 && strings.Contains(problemList(check), "leaked") { + t.Fatalf("check reported leaks although replay freed every released block:%s", problemList(check)) } repair := fsck.DefaultOptions() repair.Mode = fsck.Repair - if r := runFsck(t, remounted, repair); r.ExitCode() != fsck.ExitCorrected { - t.Fatalf("repair exit %d, want %d:%s", r.ExitCode(), fsck.ExitCorrected, problemList(r)) + wantExit := fsck.ExitClean + if len(leaked) > 0 { + wantExit = fsck.ExitCorrected + } + if r := runFsck(t, remounted, repair); r.ExitCode() != wantExit { + t.Fatalf("repair exit %d, want %d:%s", r.ExitCode(), wantExit, problemList(r)) } expectCleanFsck(t, remounted) remounted = openFormattedFS(t, path) - if got := remounted.bitmapMgr.FreeCount(); got != free+2 { - t.Fatalf("%d free blocks after repair, want %d", got, free+2) + if got := allocatedBlocks(t, remounted); !maps.Equal(got, want) { + t.Fatalf("allocated after repair %v, want %v", slices.Sorted(maps.Keys(got)), slices.Sorted(maps.Keys(want))) } + return remounted +} + +// A crash after a truncate wrote the shorter inode, but before it freed the +// blocks it cut: replay leaves those blocks allocated (or, once it frees them, +// frees them), and repair frees whatever replay left. +func TestFsckRepairReclaimsInterruptedTruncate(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + victim := createFile(t, fsys, "victim.bin") + writeAt(t, victim, pattern(1, 3*BlockSize), 0) + other := createFile(t, fsys, "other.bin") + writeAt(t, other, pattern(2, 2*BlockSize), 0) + cut := []uint64{physicalBlock(t, fsys, victim.inodeNum, 1), physicalBlock(t, fsys, victim.inodeNum, 2)} + before := allocatedBlocks(t, fsys) + + crashWhileFreeing(t, fsys, func() { tryTruncate(victim, BlockSize) }, func() bool { + ino, err := fsys.inodeTbl.Read(victim.inodeNum) + return err == nil && ino.Size == BlockSize + }) + + remounted := replayAndRepair(t, path, offset, size, before, cut) checkFile(t, lookupFile(t, remounted, "victim.bin"), pattern(1, BlockSize)) checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, 2*BlockSize)) } + +// A crash while unlinking a file in a subdirectory, after its entry is gone but +// before its blocks are freed: replay deletes the inode, and leaves the blocks +// allocated (or, once it frees them, frees them). Repair frees whatever is left. +func TestFsckRepairReclaimsInterruptedSubdirectoryUnlink(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + ctx := context.Background() + sub := newRegularFile(t, fsys) + subIno, err := fsys.updateInode(ctx, sub.inodeNum, func(ino *inode.Inode) error { + ino.Mode = inode.S_IFDIR | 0o755 + ino.LinkCount = 2 + return nil + }) + if err != nil { + t.Fatal(err) + } + if err := fsys.dirMgr.InitDirectory(ctx, subIno, uint64(inode.InodeRoot)); err != nil { + t.Fatal(err) + } + if err := fsys.inodeTbl.Write(ctx, subIno); err != nil { + t.Fatal(err) + } + root, err := fsys.inodeTbl.Read(inode.InodeRoot) + if err != nil { + t.Fatal(err) + } + if err := fsys.dirMgr.Add(ctx, root, "sub", uint64(sub.inodeNum), dir.DT_DIR); err != nil { + t.Fatal(err) + } + victim := newRegularFile(t, fsys) + if _, err := fsys.updateInode(ctx, victim.inodeNum, func(ino *inode.Inode) error { + ino.LinkCount = 1 + return nil + }); err != nil { + t.Fatal(err) + } + if subIno, err = fsys.inodeTbl.Read(sub.inodeNum); err != nil { + t.Fatal(err) + } + if err := fsys.dirMgr.Add(ctx, subIno, "victim.bin", uint64(victim.inodeNum), dir.DT_REG); err != nil { + t.Fatal(err) + } + writeAt(t, victim, pattern(1, 2*BlockSize), 0) + other := createFile(t, fsys, "other.bin") + writeAt(t, other, pattern(2, BlockSize), 0) + released := []uint64{physicalBlock(t, fsys, victim.inodeNum, 0), physicalBlock(t, fsys, victim.inodeNum, 1)} + before := allocatedBlocks(t, fsys) + + subDir := &Dir{fs: fsys, inodeNum: sub.inodeNum} + crashWhileFreeing(t, fsys, func() { subDir.Unlink(ctx, "victim.bin") }, func() bool { + ino, err := fsys.inodeTbl.Read(sub.inodeNum) + if err != nil { + return false + } + _, _, err = fsys.dirMgr.Lookup(ctx, ino, "victim.bin") + return errors.Is(err, dir.ErrNotFound) + }) + + remounted := replayAndRepair(t, path, offset, size, before, released) + subIno, err = remounted.inodeTbl.Read(sub.inodeNum) + if err != nil { + t.Fatal(err) + } + if _, _, err := remounted.dirMgr.Lookup(ctx, subIno, "victim.bin"); !errors.Is(err, dir.ErrNotFound) { + t.Fatalf("sub/victim.bin after replay: %v, want it gone", err) + } + checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, BlockSize)) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 0ad29aaa..f45e8839 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -965,11 +965,18 @@ scenarios: - id: fsck-reclaim-interrupted-truncate name: fsck -repair frees blocks an interrupted truncate leaves - description: A node crashes after a truncate writes the shorter inode but before it frees the cut blocks; replay frees nothing, check reports the blocks as leaked, repair frees them, and both files keep their data + description: A node crashes after a truncate writes the shorter inode but before it frees the cut blocks; check reports whatever cut blocks replay leaves allocated, repair frees exactly those, and both files keep their data. Passes too if replay frees them itself priority: go_only environments: go: internal/goclusterfs/fsck_reclaim_test.go::TestFsckRepairReclaimsInterruptedTruncate + - id: fsck-reclaim-interrupted-subdirectory-unlink + name: fsck -repair frees blocks an interrupted subdirectory unlink leaves + description: A node crashes while unlinking a file in a subdirectory, after the entry is gone but before the blocks are freed; replay deletes the inode, check reports whatever of its blocks replay leaves allocated, repair frees exactly those, and other files keep their data. Passes too if replay frees them itself + priority: go_only + environments: + go: internal/goclusterfs/fsck_reclaim_test.go::TestFsckRepairReclaimsInterruptedSubdirectoryUnlink + - id: fsck-reclaim-empty-sfb-containers name: fsck -repair frees SFB containers no file uses description: A file that grows out of its SFB slot leaves its container empty; check reports it as leaked, repair frees it and keeps a container another file still uses, and both files keep their data