Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions internal/fsck/fsck.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,10 +392,12 @@ type Checker struct {
snapshotData map[uint32]bool // inodes holding a snapshot's captured extents
sfbContainers map[uint64]bool // blocks holding small files of older versions
extentTrees map[uint32]*extentTree
uncounted map[uint32]int // inodes an older version shared -> problem index
refsIncomplete []string // why inodeBlocks may miss references
superblockFix func() error // runs once repair knows no node uses the filesystem
registry []byte // node registry as startRepair last saw it
uncounted map[uint32]int // inodes an older version shared -> problem index
uncountedRefs map[uint64]int // block -> references from extents an older version shared
uncountedPast map[uint32]bool // inodes an older version shared with blocks past the refcount table
refsIncomplete []string // why inodeBlocks may miss references
superblockFix func() error // runs once repair knows no node uses the filesystem
registry []byte // node registry as startRepair last saw it
}

// SuperblockData holds parsed superblock information for use by other phases.
Expand Down Expand Up @@ -465,6 +467,8 @@ func New(dev *storage.Device, opts Options) *Checker {
sfbContainers: make(map[uint64]bool),
extentTrees: make(map[uint32]*extentTree),
uncounted: make(map[uint32]int),
uncountedRefs: make(map[uint64]int),
uncountedPast: make(map[uint32]bool),
}
}

Expand Down
9 changes: 8 additions & 1 deletion internal/fsck/phase3_inodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ func (c *Checker) recordInodeExtents(inodeNum uint32, inode *InodeData) {
if ext.Length == 0 || (ext.Flags&ExtentFlagAllocated) == 0 {
continue
}
if ext.Flags&ExtentFlagShared != 0 && ext.Flags&ExtentFlagCounted == 0 {
uncounted := ext.Flags&ExtentFlagShared != 0 && ext.Flags&ExtentFlagCounted == 0
if uncounted {
c.uncounted[inodeNum] = -1
}

Expand All @@ -314,6 +315,12 @@ func (c *Checker) recordInodeExtents(inodeNum uint32, inode *InodeData) {
for j := uint64(0); j < ext.Length; j++ {
blockNum := ext.PhysicalBlock + j
c.inodeBlocks[blockNum]++
if uncounted {
c.uncountedRefs[blockNum]++
}
}
if uncounted && ext.PhysicalBlock+ext.Length > c.refcountCoverage() {
c.uncountedPast[inodeNum] = true
}
}
}
Expand Down
34 changes: 34 additions & 0 deletions internal/fsck/phase5_refcounts.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package fsck
import (
"encoding/binary"
"fmt"
"slices"
"strings"

"github.com/supporttools/GoClusterFS/internal/storage"
Expand Down Expand Up @@ -58,6 +59,8 @@ type refcountFix struct {
// fewer would free a block another inode still maps
// - Blocks count no more owners than inodes map them; more would leak them
// - No refcount overflow (0xFFFFFFFF)
// - No block past the table, which has no count for it, is mapped by
// several inodes that count their sharing
//
// In repair mode, and only when every inode's extents were read, the counts
// are rebuilt from those references, and extents an older version shared
Expand Down Expand Up @@ -111,6 +114,11 @@ func (c *Checker) checkRefcounts() error {
}
}
}
if covered := c.refcountCoverage(); sb.TotalBlocks > covered {
c.addProblem(PhaseRefcounts, Warning, "refcount table",
fmt.Sprintf("has entries for blocks 0-%d of %d; snapshots and clones of files with data in higher blocks fail", covered-1, sb.TotalBlocks), false)
c.checkSharingPastTable(covered)
}
c.reportUncounted()

c.logf(" Shared blocks (refcount > 1): %d\n", sharedBlocks)
Expand Down Expand Up @@ -143,6 +151,32 @@ func (c *Checker) checkRefcounts() error {
return c.markSharingCounted()
}

// refcountCoverage returns how many blocks the refcount table has entries for.
func (c *Checker) refcountCoverage() uint64 {
return c.sbData.RefCountTableSize / RefcountSectorSize * RefcountEntriesPerSector
}

// checkSharingPastTable reports blocks past the refcount table that inodes share
// with counts, which the first delete frees. Only-uncounted sharing is never freed.
func (c *Checker) checkSharingPastTable(covered uint64) {
var blocks []uint64
for block, refs := range c.inodeBlocks {
if block >= covered && refs > 1 && c.uncountedRefs[block] < refs {
blocks = append(blocks, block)
}
}
slices.Sort(blocks)
for _, block := range blocks[:min(len(blocks), 10)] {
c.addProblem(PhaseRefcounts, Critical, fmt.Sprintf("block %d", block),
fmt.Sprintf("mapped by %d inodes but past the refcount table, which has no count for it: deleting one of them frees a block the others still map",
c.inodeBlocks[block]), false)
}
if len(blocks) > 10 {
c.addProblem(PhaseRefcounts, Critical, "blocks",
fmt.Sprintf("%d more blocks past the refcount table are mapped by several inodes", len(blocks)-10), false)
}
}

// checkRefcount compares one block's count with the inodes that map it. It
// reports a problem and returns the count repair should set, if any.
func (c *Checker) checkRefcount(block uint64, count uint32) (uint32, bool) {
Expand Down
9 changes: 9 additions & 0 deletions internal/fsck/repair.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ func (c *Checker) reportUncounted() {
}
slices.Sort(nums)
for _, num := range nums {
// Marked counted, blocks past the table would be freed on their first release.
if c.uncountedPast[num] {
c.addProblem(PhaseRefcounts, Error, fmt.Sprintf("inode %d", num),
"shares blocks past the refcount table the way older versions did, without counting the owners; repair cannot count them, "+
"so writes to it change the other files that map those blocks. Copy the file and delete the original to separate it",
false)
delete(c.uncounted, num)
continue
}
c.addProblem(PhaseRefcounts, Warning, fmt.Sprintf("inode %d", num),
"shares blocks the way older versions did, without counting the owners; snapshots and clones of it fail until repaired",
true)
Expand Down
79 changes: 78 additions & 1 deletion internal/goclusterfs/fsck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/binary"
"fmt"
"os"
"path/filepath"
"strings"
Expand All @@ -23,12 +24,17 @@ import (

// newFormattedImage formats an image the way goclusterfs-mount -format does.
func newFormattedImage(t *testing.T) string {
t.Helper()
return newFormattedImageOfSize(t, superblock.MinDeviceSize+64<<20)
}

func newFormattedImageOfSize(t *testing.T, size int64) string {
t.Helper()
path := filepath.Join(t.TempDir(), "fs.img")
if err := os.WriteFile(path, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Truncate(path, superblock.MinDeviceSize+64<<20); err != nil {
if err := os.Truncate(path, size); err != nil {
t.Fatal(err)
}
dev, err := storage.Open(path)
Expand Down Expand Up @@ -298,6 +304,77 @@ func TestFsckRepairsOlderSharing(t *testing.T) {
}
}

// On a 200 GiB volume the 1 MB refcount table has entries for blocks 0-130815
// only. Several inodes counting on a share of a higher block lose data when
// one is deleted, and marking older sharing of such blocks counted would
// cause the same, so repair leaves that sharing alone.
func TestFsckReportsSharingPastRefcountTable(t *testing.T) {
path := newFormattedImageOfSize(t, 200<<30)
fsys := openFormattedFS(t, path)
ctx := context.Background()
sb, err := superblock.Read(fsys.dev)
if err != nil {
t.Fatal(err)
}
covered := sb.RefCountTableSize / 4096 * 511
counted, older := covered+5, covered+9

share := func(name string, block uint64, flags uint32) *File {
f := createFile(t, fsys, name)
if _, err := fsys.updateInode(ctx, f.inodeNum, func(ino *inode.Inode) error {
ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: block, Length: 1, Flags: extent.FlagAllocated | flags}
ino.ExtentCount, ino.Size, ino.AllocatedBlocks, ino.WrittenBlocks = 1, 4096, 1, 1
return nil
}); err != nil {
t.Fatal(err)
}
return f
}
for _, block := range []uint64{counted, older} {
if err := fsys.bitmapMgr.MarkAllocated(ctx, block); err != nil {
t.Fatal(err)
}
}
share("counted-a", counted, extent.FlagShared|extent.FlagCounted)
share("counted-b", counted, extent.FlagShared|extent.FlagCounted)
oldA := share("older-a", older, extent.FlagShared)
share("older-b", older, extent.FlagShared)

check := runFsck(t, fsys, fsck.DefaultOptions())
problems := problemList(check)
for _, want := range []string{
fmt.Sprintf("has entries for blocks 0-%d of %d", covered-1, sb.TotalBlocks),
fmt.Sprintf("block %d: mapped by 2 inodes but past the refcount table", counted),
fmt.Sprintf("inode %d: shares blocks past the refcount table the way older versions did", oldA.inodeNum),
} {
if !strings.Contains(problems, want) {
t.Errorf("fsck did not report %q:%s", want, problems)
}
}
if strings.Contains(problems, fmt.Sprintf("block %d: mapped by", older)) {
t.Errorf("block %d is only shared without counts, so it is never freed, yet fsck reported it:%s", older, problems)
}
if check.ExitCode() != fsck.ExitUncorrected {
t.Errorf("check exit %d, want %d", check.ExitCode(), fsck.ExitUncorrected)
}
for _, p := range check.Problems {
if p.Location == fmt.Sprintf("inode %d", oldA.inodeNum) && p.Severity != fsck.Error {
t.Errorf("inode %d reported as %v, want an error: writes to it change older-b too", oldA.inodeNum, p.Severity)
}
}

repair := fsck.DefaultOptions()
repair.Mode = fsck.Repair
runFsck(t, fsys, repair)
ino, err := fsys.inodeTbl.Read(oldA.inodeNum)
if err != nil {
t.Fatal(err)
}
if ino.InlineExtents[0].Flags&extent.FlagCounted != 0 {
t.Fatalf("repair marked older sharing of block %d counted, so its first release would free it", older)
}
}

// The runtime cannot read a refcount sector with a bad checksum. Repair
// rebuilds every count in it from the extents that map the blocks.
func TestFsckRebuildsCorruptRefcountSector(t *testing.T) {
Expand Down
7 changes: 7 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,13 @@ scenarios:
environments:
go: internal/goclusterfs/fsck_test.go::TestFsckRepairsOlderSharing

- id: fsck-sharing-past-refcount-table
name: fsck reports sharing the refcount table cannot count
description: On a 200 GiB volume fsck warns that the refcount table covers blocks 0-130815, reports Critical for a higher block that several inodes share with counts, and repair leaves older uncounted sharing of such blocks uncounted
priority: go_only
environments:
go: internal/goclusterfs/fsck_test.go::TestFsckReportsSharingPastRefcountTable

- id: fsck-rebuild-refcount-sector
name: fsck -repair rebuilds a corrupt refcount sector
description: A refcount sector with a bad checksum is rebuilt from the extents that map its blocks
Expand Down
Loading