diff --git a/internal/fsck/fsck.go b/internal/fsck/fsck.go index 1f59e4f7..472f3d78 100644 --- a/internal/fsck/fsck.go +++ b/internal/fsck/fsck.go @@ -392,10 +392,12 @@ type Checker struct { snapshotData map[uint32]bool // inodes holding a snapshot's captured extents sfbContainers map[uint64]bool // blocks holding small files of older versions extentTrees map[uint32]*extentTree - uncounted map[uint32]int // inodes an older version shared -> problem index - refsIncomplete []string // why inodeBlocks may miss references - superblockFix func() error // runs once repair knows no node uses the filesystem - registry []byte // node registry as startRepair last saw it + uncounted map[uint32]int // inodes an older version shared -> problem index + uncountedRefs map[uint64]int // block -> references from extents an older version shared + uncountedPast map[uint32]bool // inodes an older version shared with blocks past the refcount table + refsIncomplete []string // why inodeBlocks may miss references + superblockFix func() error // runs once repair knows no node uses the filesystem + registry []byte // node registry as startRepair last saw it } // SuperblockData holds parsed superblock information for use by other phases. @@ -465,6 +467,8 @@ func New(dev *storage.Device, opts Options) *Checker { sfbContainers: make(map[uint64]bool), extentTrees: make(map[uint32]*extentTree), uncounted: make(map[uint32]int), + uncountedRefs: make(map[uint64]int), + uncountedPast: make(map[uint32]bool), } } diff --git a/internal/fsck/phase3_inodes.go b/internal/fsck/phase3_inodes.go index 17ebfedb..38f38e7c 100644 --- a/internal/fsck/phase3_inodes.go +++ b/internal/fsck/phase3_inodes.go @@ -297,7 +297,8 @@ func (c *Checker) recordInodeExtents(inodeNum uint32, inode *InodeData) { if ext.Length == 0 || (ext.Flags&ExtentFlagAllocated) == 0 { continue } - if ext.Flags&ExtentFlagShared != 0 && ext.Flags&ExtentFlagCounted == 0 { + uncounted := ext.Flags&ExtentFlagShared != 0 && ext.Flags&ExtentFlagCounted == 0 + if uncounted { c.uncounted[inodeNum] = -1 } @@ -314,6 +315,12 @@ func (c *Checker) recordInodeExtents(inodeNum uint32, inode *InodeData) { for j := uint64(0); j < ext.Length; j++ { blockNum := ext.PhysicalBlock + j c.inodeBlocks[blockNum]++ + if uncounted { + c.uncountedRefs[blockNum]++ + } + } + if uncounted && ext.PhysicalBlock+ext.Length > c.refcountCoverage() { + c.uncountedPast[inodeNum] = true } } } diff --git a/internal/fsck/phase5_refcounts.go b/internal/fsck/phase5_refcounts.go index 82aa1060..2cdfbbca 100644 --- a/internal/fsck/phase5_refcounts.go +++ b/internal/fsck/phase5_refcounts.go @@ -3,6 +3,7 @@ package fsck import ( "encoding/binary" "fmt" + "slices" "strings" "github.com/supporttools/GoClusterFS/internal/storage" @@ -58,6 +59,8 @@ type refcountFix struct { // fewer would free a block another inode still maps // - Blocks count no more owners than inodes map them; more would leak them // - No refcount overflow (0xFFFFFFFF) +// - No block past the table, which has no count for it, is mapped by +// several inodes that count their sharing // // In repair mode, and only when every inode's extents were read, the counts // are rebuilt from those references, and extents an older version shared @@ -111,6 +114,11 @@ func (c *Checker) checkRefcounts() error { } } } + if covered := c.refcountCoverage(); sb.TotalBlocks > covered { + c.addProblem(PhaseRefcounts, Warning, "refcount table", + fmt.Sprintf("has entries for blocks 0-%d of %d; snapshots and clones of files with data in higher blocks fail", covered-1, sb.TotalBlocks), false) + c.checkSharingPastTable(covered) + } c.reportUncounted() c.logf(" Shared blocks (refcount > 1): %d\n", sharedBlocks) @@ -143,6 +151,32 @@ func (c *Checker) checkRefcounts() error { return c.markSharingCounted() } +// refcountCoverage returns how many blocks the refcount table has entries for. +func (c *Checker) refcountCoverage() uint64 { + return c.sbData.RefCountTableSize / RefcountSectorSize * RefcountEntriesPerSector +} + +// checkSharingPastTable reports blocks past the refcount table that inodes share +// with counts, which the first delete frees. Only-uncounted sharing is never freed. +func (c *Checker) checkSharingPastTable(covered uint64) { + var blocks []uint64 + for block, refs := range c.inodeBlocks { + if block >= covered && refs > 1 && c.uncountedRefs[block] < refs { + blocks = append(blocks, block) + } + } + slices.Sort(blocks) + for _, block := range blocks[:min(len(blocks), 10)] { + c.addProblem(PhaseRefcounts, Critical, fmt.Sprintf("block %d", block), + fmt.Sprintf("mapped by %d inodes but past the refcount table, which has no count for it: deleting one of them frees a block the others still map", + c.inodeBlocks[block]), false) + } + if len(blocks) > 10 { + c.addProblem(PhaseRefcounts, Critical, "blocks", + fmt.Sprintf("%d more blocks past the refcount table are mapped by several inodes", len(blocks)-10), false) + } +} + // checkRefcount compares one block's count with the inodes that map it. It // reports a problem and returns the count repair should set, if any. func (c *Checker) checkRefcount(block uint64, count uint32) (uint32, bool) { diff --git a/internal/fsck/repair.go b/internal/fsck/repair.go index 4bb3979c..5afbfec5 100644 --- a/internal/fsck/repair.go +++ b/internal/fsck/repair.go @@ -59,6 +59,15 @@ func (c *Checker) reportUncounted() { } slices.Sort(nums) for _, num := range nums { + // Marked counted, blocks past the table would be freed on their first release. + if c.uncountedPast[num] { + c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("inode %d", num), + "shares blocks past the refcount table the way older versions did, without counting the owners; repair cannot count them, "+ + "so writes to it change the other files that map those blocks. Copy the file and delete the original to separate it", + false) + delete(c.uncounted, num) + continue + } c.addProblem(PhaseRefcounts, Warning, fmt.Sprintf("inode %d", num), "shares blocks the way older versions did, without counting the owners; snapshots and clones of it fail until repaired", true) diff --git a/internal/goclusterfs/fsck_test.go b/internal/goclusterfs/fsck_test.go index a16355fc..c4aa3566 100644 --- a/internal/goclusterfs/fsck_test.go +++ b/internal/goclusterfs/fsck_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "fmt" "os" "path/filepath" "strings" @@ -23,12 +24,17 @@ import ( // newFormattedImage formats an image the way goclusterfs-mount -format does. func newFormattedImage(t *testing.T) string { + t.Helper() + return newFormattedImageOfSize(t, superblock.MinDeviceSize+64<<20) +} + +func newFormattedImageOfSize(t *testing.T, size int64) string { t.Helper() path := filepath.Join(t.TempDir(), "fs.img") if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } - if err := os.Truncate(path, superblock.MinDeviceSize+64<<20); err != nil { + if err := os.Truncate(path, size); err != nil { t.Fatal(err) } dev, err := storage.Open(path) @@ -298,6 +304,77 @@ func TestFsckRepairsOlderSharing(t *testing.T) { } } +// On a 200 GiB volume the 1 MB refcount table has entries for blocks 0-130815 +// only. Several inodes counting on a share of a higher block lose data when +// one is deleted, and marking older sharing of such blocks counted would +// cause the same, so repair leaves that sharing alone. +func TestFsckReportsSharingPastRefcountTable(t *testing.T) { + path := newFormattedImageOfSize(t, 200<<30) + fsys := openFormattedFS(t, path) + ctx := context.Background() + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + covered := sb.RefCountTableSize / 4096 * 511 + counted, older := covered+5, covered+9 + + share := func(name string, block uint64, flags uint32) *File { + f := createFile(t, fsys, name) + if _, err := fsys.updateInode(ctx, f.inodeNum, func(ino *inode.Inode) error { + ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: block, Length: 1, Flags: extent.FlagAllocated | flags} + ino.ExtentCount, ino.Size, ino.AllocatedBlocks, ino.WrittenBlocks = 1, 4096, 1, 1 + return nil + }); err != nil { + t.Fatal(err) + } + return f + } + for _, block := range []uint64{counted, older} { + if err := fsys.bitmapMgr.MarkAllocated(ctx, block); err != nil { + t.Fatal(err) + } + } + share("counted-a", counted, extent.FlagShared|extent.FlagCounted) + share("counted-b", counted, extent.FlagShared|extent.FlagCounted) + oldA := share("older-a", older, extent.FlagShared) + share("older-b", older, extent.FlagShared) + + check := runFsck(t, fsys, fsck.DefaultOptions()) + problems := problemList(check) + for _, want := range []string{ + fmt.Sprintf("has entries for blocks 0-%d of %d", covered-1, sb.TotalBlocks), + fmt.Sprintf("block %d: mapped by 2 inodes but past the refcount table", counted), + fmt.Sprintf("inode %d: shares blocks past the refcount table the way older versions did", oldA.inodeNum), + } { + if !strings.Contains(problems, want) { + t.Errorf("fsck did not report %q:%s", want, problems) + } + } + if strings.Contains(problems, fmt.Sprintf("block %d: mapped by", older)) { + t.Errorf("block %d is only shared without counts, so it is never freed, yet fsck reported it:%s", older, problems) + } + if check.ExitCode() != fsck.ExitUncorrected { + t.Errorf("check exit %d, want %d", check.ExitCode(), fsck.ExitUncorrected) + } + for _, p := range check.Problems { + if p.Location == fmt.Sprintf("inode %d", oldA.inodeNum) && p.Severity != fsck.Error { + t.Errorf("inode %d reported as %v, want an error: writes to it change older-b too", oldA.inodeNum, p.Severity) + } + } + + repair := fsck.DefaultOptions() + repair.Mode = fsck.Repair + runFsck(t, fsys, repair) + ino, err := fsys.inodeTbl.Read(oldA.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.InlineExtents[0].Flags&extent.FlagCounted != 0 { + t.Fatalf("repair marked older sharing of block %d counted, so its first release would free it", older) + } +} + // The runtime cannot read a refcount sector with a bad checksum. Repair // rebuilds every count in it from the extents that map the blocks. func TestFsckRebuildsCorruptRefcountSector(t *testing.T) { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 4e46d18f..a3c01e7c 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -837,6 +837,13 @@ scenarios: environments: go: internal/goclusterfs/fsck_test.go::TestFsckRepairsOlderSharing + - id: fsck-sharing-past-refcount-table + name: fsck reports sharing the refcount table cannot count + description: On a 200 GiB volume fsck warns that the refcount table covers blocks 0-130815, reports Critical for a higher block that several inodes share with counts, and repair leaves older uncounted sharing of such blocks uncounted + priority: go_only + environments: + go: internal/goclusterfs/fsck_test.go::TestFsckReportsSharingPastRefcountTable + - id: fsck-rebuild-refcount-sector name: fsck -repair rebuilds a corrupt refcount sector description: A refcount sector with a bad checksum is rebuilt from the extents that map its blocks