From 01b448c3ea98b19931fd0a13190dbda90155d0da Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:42:17 -0500 Subject: [PATCH 1/3] feat(fsck): list node registry slots and release a dead node's slot A dead node's slot whose journal header is damaged was never freed: peers fail to replay it every minute, and only the same host before a reboot could take it back. goclusterfs-fsck -list-slots shows each slot's owner, lease and journal state. -release-slot N checks that no node runs and that the slot's lease ended a minute ago, and prints what releasing discards; with -discard-journal it resets the slot's journal and frees the slot, so the next mount can use it. cmd/mount is unchanged. Co-Authored-By: Claude Opus 5.5 --- cmd/fsck/main.go | 36 +++++++ cmd/fsck/slots.go | 92 +++++++++++++++++ docs/OPERATIONS.md | 10 +- internal/fsck/repair.go | 2 +- internal/fsck/slots.go | 178 +++++++++++++++++++++++++++++++++ internal/fsck/slots_test.go | 146 +++++++++++++++++++++++++++ internal/node/liveness.go | 13 +++ internal/node/registry.go | 31 ++++++ tests/integration/fsck_test.go | 69 +++++++++++++ tests/test-manifest.yaml | 24 +++++ 10 files changed, 599 insertions(+), 2 deletions(-) create mode 100644 cmd/fsck/slots.go create mode 100644 internal/fsck/slots.go create mode 100644 internal/fsck/slots_test.go diff --git a/cmd/fsck/main.go b/cmd/fsck/main.go index 0fe457f2..ba9fe12b 100644 --- a/cmd/fsck/main.go +++ b/cmd/fsck/main.go @@ -14,6 +14,9 @@ // -repair -dry-run: Report what -repair would change, write nothing // -force: Keep checking after a fatal error; with -repair, repair even if // a stopped node's journal still needs replay +// -list-slots: List node registry slots with their lease and journal +// -release-slot N: Free the slot of a node that will not return; with +// -discard-journal, reset its journal and free it // // Exit codes, bits as with fsck(8): // @@ -47,6 +50,10 @@ func main() { summaryOnly = flag.Bool("summary", false, "Only print final summary") showProblems = flag.Bool("show-problems", true, "Show individual problems found") showVersion = flag.Bool("version", false, "Print version and exit") + + listSlots = flag.Bool("list-slots", false, "List node registry slots with their lease and journal, then exit") + releaseSlotN = flag.Int("release-slot", -1, "Check that registry slot N can be released and print what releasing it discards") + discardJournal = flag.Bool("discard-journal", false, "With -release-slot, reset the slot's journal and free the slot") ) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n\n", os.Args[0]) @@ -58,7 +65,10 @@ func main() { fmt.Fprintf(os.Stderr, " %s -repair /path/to/disk.img # Check and repair\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s -verbose -repair /dev/sdc # Verbose repair\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s -repair -dry-run /dev/sdc # Show what repair would change\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s -list-slots /dev/sdb # Show node registry slots\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s -release-slot 3 /dev/sdb # Show what releasing slot 3 discards\n", os.Args[0]) fmt.Fprint(os.Stderr, repairHelp) + fmt.Fprint(os.Stderr, slotsHelp) fmt.Fprintf(os.Stderr, "\nExit codes (bits, as with fsck(8); 5 means some errors were repaired and some remain):\n") fmt.Fprintf(os.Stderr, " 0: No errors (warnings may remain)\n") fmt.Fprintf(os.Stderr, " 1: Errors were repaired\n") @@ -88,6 +98,18 @@ func main() { fmt.Fprintln(os.Stderr, "Error: -dry-run needs -repair") os.Exit(fsck.ExitUsage) } + releasing := *releaseSlotN != -1 + switch { + case *discardJournal && !releasing: + fmt.Fprintln(os.Stderr, "Error: -discard-journal needs -release-slot") + os.Exit(fsck.ExitUsage) + case releasing && *releaseSlotN < 0: + fmt.Fprintln(os.Stderr, "Error: -release-slot needs a slot number") + os.Exit(fsck.ExitUsage) + case *listSlots && releasing, (*listSlots || releasing) && *repair: + fmt.Fprintln(os.Stderr, "Error: -list-slots, -release-slot and -repair run separately") + os.Exit(fsck.ExitUsage) + } devicePath := flag.Arg(0) // Determine mode @@ -111,6 +133,20 @@ func main() { } defer dev.Close() + if *listSlots || releasing { + checker := fsck.New(dev, fsck.Options{Output: os.Stdout, Verbose: *verbose}) + if *listSlots { + err = printSlots(os.Stdout, checker) + } else { + err = releaseSlot(os.Stdout, checker, *releaseSlotN, *discardJournal) + } + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(fsck.ExitOperational) + } + os.Exit(fsck.ExitClean) + } + // Configure fsck options opts := fsck.Options{ Mode: mode, diff --git a/cmd/fsck/slots.go b/cmd/fsck/slots.go new file mode 100644 index 00000000..af2759ce --- /dev/null +++ b/cmd/fsck/slots.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "io" + "strings" + "text/tabwriter" + "time" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/node" +) + +const slotsHelp = ` +Node registry slots: + Every mounted node holds a registry slot with its own journal. A crashed + node's slot frees once another node replays its journal, or when the same + host mounts again. -list-slots shows each slot that is in use or whose + journal needs attention. + + -release-slot N frees the slot of a node that will not return, for example + one whose journal header is damaged. It refuses while any node has the + filesystem mounted, or until a minute after the slot's lease ended, and + prints what releasing would discard. Add -discard-journal to reset the + slot's journal and free the slot. +` + +// printSlots lists the slots that are in use or whose journal was written +// and is not clean. +func printSlots(out io.Writer, checker *fsck.Checker) error { + slots, err := checker.Slots() + if err != nil { + return err + } + w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "SLOT\tSTATE\tHOST\tNODE\tLEASE\tJOURNAL") + shown := 0 + for _, s := range slots { + if !s.InUse() && s.JournalFresh { + continue + } + state, host, uuid := "unreadable: "+fmt.Sprint(s.Err), "-", "-" + if s.Slot != nil { + state = s.Slot.State.String() + host = strings.TrimRight(string(s.Slot.Hostname[:]), "\x00") + uuid = fmt.Sprintf("%x", s.Slot.UUID[:8]) + } + fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\n", s.Index, state, host, uuid, lease(s), s.Journal) + shown++ + } + if err := w.Flush(); err != nil { + return err + } + fmt.Fprintf(out, "\n%d of %d slots shown; the others are free, with a journal that is clean or was never used.\n", shown, len(slots)) + return nil +} + +// releaseSlot checks slot i and prints what releasing it discards. With +// discard set it also releases the slot. +func releaseSlot(out io.Writer, checker *fsck.Checker, i int, discard bool) error { + plan, err := checker.PlanRelease(i) + if err != nil { + return err + } + s := plan.Slot + fmt.Fprintf(out, "Slot %d: %s, %s, lease %s\n", s.Index, s.Name(), s.Slot.State, lease(s)) + fmt.Fprintf(out, "Journal: %s\n", s.Journal) + fmt.Fprintf(out, "Releasing discards %s.\n", plan.Discards) + if !discard { + fmt.Fprintln(out, "Nothing was written. Run again with -discard-journal to reset the journal and free the slot.") + return nil + } + if err := checker.Release(plan); err != nil { + return err + } + fmt.Fprintf(out, "Released slot %d: its journal is reset and a node can register in it.\n", i) + return nil +} + +func lease(s fsck.SlotStatus) string { + switch { + case s.Running: + return "held (node running)" + case s.Slot == nil || s.Slot.State == node.NodeStateEmpty: + return "-" + } + expiry := s.Slot.LeaseExpiry() + if left := time.Until(expiry); left > 0 { + return "ends in " + left.Round(time.Second).String() + } + return "ended " + time.Since(expiry).Round(time.Second).String() + " ago" +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 6e7a8008..417181aa 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -288,7 +288,15 @@ A mount exits with `Journal recovery for slot N failed: ...` when it cannot repl `-skip-recovery` does not stop the node from replaying crashed peers' journals. `-journal=false` also skips the replay of its own slot, and leaves that journal in place for the next mount, so don't use it on a device that may have a dirty journal. -A slot belongs to its host's hostname and boot ID. If the damaged slot belongs to a node that has rebooted or will not mount again, the other nodes log `Replaying slot N ... failed` and retry every minute. Between retries, a new mount on any host can take that slot and then fail with the same error. There is no safe tool to free the slot yet. +A slot belongs to its host's hostname and boot ID. If the damaged slot belongs to a node that has rebooted or will not mount again, the other nodes log `Replaying slot N ... failed` and retry every minute. Between retries, a new mount on any host can take that slot and then fail with the same error. Free the slot with fsck. Unmount the filesystem on every node first, then: + +```bash +./goclusterfs-fsck -list-slots /dev/device # find the slot and its journal state +./goclusterfs-fsck -release-slot N /dev/device # check the slot; prints what releasing discards +./goclusterfs-fsck -release-slot N -discard-journal /dev/device # reset its journal and free the slot +``` + +`-release-slot` refuses while any node has the filesystem mounted, and until a minute after the slot's lease ended: its last heartbeat plus 10 seconds, or for a slot another node was replaying, that node's last lease renewal plus 60 seconds. Without `-force` it also refuses a journal that a mount might still replay: one with committed transactions that fsck reads in full, or one fsck cannot read to the end. Mount a node to replay it instead, and add `-force` only if that replay fails. With `-discard-journal`, fsck holds the slot's recovery lease while it resets the journal, frees the slot, and exits 1. The next node to mount can then use the slot. Run fsck afterwards, because discarded transactions may have been only partly applied. ### Emergency: All Nodes Down diff --git a/internal/fsck/repair.go b/internal/fsck/repair.go index 5afbfec5..658c12b5 100644 --- a/internal/fsck/repair.go +++ b/internal/fsck/repair.go @@ -287,7 +287,7 @@ func (c *Checker) journalNeedsReplay(i int) string { if sb.MaxNodes == 0 || uint32(i) >= sb.MaxNodes { return "has no journal slot" } - off := sb.JournalRegionOffset + uint64(i)*(sb.JournalRegionSize/uint64(sb.MaxNodes)) + off, _ := c.journalArea(i) buf := storage.NewAlignedBuffer(storage.SectorSize).Bytes() if err := c.readSector(buf, off); err != nil { return "has an unreadable journal: " + err.Error() diff --git a/internal/fsck/slots.go b/internal/fsck/slots.go new file mode 100644 index 00000000..bfb9118c --- /dev/null +++ b/internal/fsck/slots.go @@ -0,0 +1,178 @@ +package fsck + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +// ReleaseMargin is how long a slot's lease must have been over before fsck +// releases the slot, so a node that is only slow, or whose clock lags, keeps it. +const ReleaseMargin = time.Minute + +// SlotStatus describes a node registry slot and its journal. +type SlotStatus struct { + Index int + Slot *node.Slot // nil when the slot is unreadable + Err error // why the slot is unreadable + Running bool // the slot changed while fsck watched the registry + Journal string // the journal's state, or why it cannot be read + // JournalFresh means the journal was never initialized or is clean. + JournalFresh bool + + raw []byte +} + +// InUse reports whether the slot belongs to a node or is unreadable. +func (s SlotStatus) InUse() bool { + return s.Slot == nil || s.Slot.State != node.NodeStateEmpty +} + +// Name names the slot's node. +func (s SlotStatus) Name() string { + return slotName(s.Index, s.Slot) +} + +// Slots reads the superblock and describes every node registry slot. It first +// watches the registry for running nodes, for up to 11 seconds when a +// heartbeat is recent. +func (c *Checker) Slots() ([]SlotStatus, error) { + if err := c.checkSuperblock(); err != nil { + return nil, fmt.Errorf("superblock: %w", err) + } + slots, _ := node.NewRegistry(c.dev, c.sbData.NodeRegistryOffset).Slots() + running, err := c.watchRegistry(slots) + if err != nil { + return nil, fmt.Errorf("read the node registry: %w", err) + } + + status := make([]SlotStatus, min(len(c.registry)/node.SlotSize, int(c.sbData.MaxNodes))) + for i := range status { + raw := c.registry[i*node.SlotSize : (i+1)*node.SlotSize] + s := SlotStatus{Index: i, Running: running[i], raw: raw} + s.Slot, s.Err = node.ParseSlot(raw) + region, err := c.slotJournal(i) + s.Journal = journalState(region, err) + s.JournalFresh = errors.Is(err, journal.ErrJournalUninitialized) || + err == nil && region.State() == journal.JournalStateClean + status[i] = s + } + return status, nil +} + +// ReleasePlan is a checked request to release one node registry slot. +type ReleasePlan struct { + Slot SlotStatus + Discards string // what resetting the slot's journal throws away +} + +// PlanRelease checks that slot i can be released: no node runs, the slot is +// readable, and its lease ended ReleaseMargin ago. It writes nothing. +func (c *Checker) PlanRelease(i int) (*ReleasePlan, error) { + slots, err := c.Slots() + if err != nil { + return nil, err + } + if i < 0 || i >= len(slots) { + return nil, fmt.Errorf("slot %d does not exist; slots are 0 to %d", i, len(slots)-1) + } + for _, s := range slots { + if s.Running { + return nil, fmt.Errorf("release refused: the filesystem is mounted by %s; unmount it on every node first", s.Name()) + } + } + s := slots[i] + if s.Err != nil { + return nil, fmt.Errorf("release refused: slot %d is unreadable (%v)", i, s.Err) + } + if after := s.Slot.LeaseExpiry().Add(ReleaseMargin); time.Now().Before(after) { + return nil, fmt.Errorf("release refused: %s may still be alive; run again after %s", + s.Name(), after.Format(time.DateTime)) + } + return &ReleasePlan{Slot: s, Discards: discards(c.slotJournal(i))}, nil +} + +// Release resets the plan's journal and empties its registry slot, so a node +// can register there with a new journal. It stops if any registry slot +// changed since PlanRelease watched the registry. +func (c *Checker) Release(plan *ReleasePlan) error { + i := plan.Slot.Index + now, err := c.readRegistry() + if err != nil { + return fmt.Errorf("read the node registry: %w", err) + } + if changed := changedSlots(c.registry, now); len(changed) > 0 { + return fmt.Errorf("release stopped: node registry slot %d changed since fsck checked it; run again", changed[0]) + } + + offset, _ := c.journalArea(i) + if _, err := c.dev.WriteAt(storage.NewAlignedBuffer(storage.SectorSize).Bytes(), int64(offset)); err != nil { + return fmt.Errorf("reset the journal of slot %d: %w", i, err) + } + if err := c.dev.Sync(); err != nil { + return fmt.Errorf("sync: %w", err) + } + released, err := node.NewRegistry(c.dev, c.sbData.NodeRegistryOffset).ReleaseSlot(i, plan.Slot.raw) + if err != nil { + return fmt.Errorf("release slot %d: %w", i, err) + } + if !released { + return fmt.Errorf("slot %d changed while fsck released it: its journal is reset but the slot is not; run again", i) + } + return c.dev.Sync() +} + +// journalArea returns where slot i's journal starts and how long it is. +func (c *Checker) journalArea(i int) (offset, size uint64) { + size = c.sbData.JournalRegionSize / uint64(c.sbData.MaxNodes) + return c.sbData.JournalRegionOffset + uint64(i)*size, size +} + +// slotJournal opens slot i's journal. A journal that was never initialized +// returns journal.ErrJournalUninitialized. +func (c *Checker) slotJournal(i int) (*journal.Region, error) { + offset, size := c.journalArea(i) + return journal.OpenRegionForRecovery(c.dev, offset, size) +} + +func journalState(region *journal.Region, err error) string { + switch { + case errors.Is(err, journal.ErrJournalUninitialized): + return "never initialized" + case err != nil: + return "damaged: " + rootCause(err).Error() + } + return strings.ToLower(region.State().String()) +} + +// discards says what resetting a journal throws away. +func discards(region *journal.Region, err error) string { + switch { + case errors.Is(err, journal.ErrJournalUninitialized): + return "nothing: the journal was never initialized" + case err != nil: + return "the whole journal, unread: " + rootCause(err).Error() + case region.State() == journal.JournalStateClean: + return "nothing: the journal is clean" + case !region.NeedsRecovery(): + return "the whole journal, unread: its state is " + strings.ToLower(region.State().String()) + } + txns, _, err := journal.DryRunRecover(region) + what := fmt.Sprintf("%d committed transactions that were never replayed", len(txns)) + if err != nil { + what += fmt.Sprintf(", and whatever follows a record fsck cannot read (%v)", err) + } + return what +} + +func rootCause(err error) error { + if inner := errors.Unwrap(err); inner != nil { + return inner + } + return err +} diff --git a/internal/fsck/slots_test.go b/internal/fsck/slots_test.go new file mode 100644 index 00000000..d404bd19 --- /dev/null +++ b/internal/fsck/slots_test.go @@ -0,0 +1,146 @@ +package fsck_test + +import ( + "bytes" + "context" + "encoding/binary" + "strings" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +func TestReleaseSlot(t *testing.T) { + t.Cleanup(fsck.SetHeartbeatWatch(2*node.HeartbeatInterval, 2*node.HeartbeatInterval)) + var sb *superblock.Superblock + dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) + ctx := context.Background() + checker := func() *fsck.Checker { return fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) } + refused := func(slot int, reason string) { + t.Helper() + if _, err := checker().PlanRelease(slot); err == nil || !strings.Contains(err.Error(), reason) { + t.Fatalf("PlanRelease(%d) error = %v, want one containing %q", slot, err, reason) + } + } + + gone := node.NewRegistry(dev, sb.NodeRegistryOffset) + goneSlot, err := gone.Register(ctx, "gone") + if err != nil { + t.Fatal(err) + } + if err := gone.StartHeartbeat(); err != nil { + t.Fatal(err) + } + live := node.NewRegistry(dev, sb.NodeRegistryOffset) + if _, err := live.Register(ctx, "live"); err != nil { + t.Fatal(err) + } + if err := live.StartHeartbeat(); err != nil { + t.Fatal(err) + } + gone.Stop() + refused(goneSlot, "mounted by node 1 (live)") + if err := live.Unregister(ctx); err != nil { + t.Fatal(err) + } + refused(goneSlot, "node 0 (gone) may still be alive") + + editSlot(t, dev, sb, goneSlot, func(slot []byte) { + binary.LittleEndian.PutUint32(slot[4:8], uint32(node.NodeStateDead)) + binary.LittleEndian.PutUint64(slot[172:180], uint64(time.Now().Unix())) + }) + refused(goneSlot, "may still be alive") + + editSlot(t, dev, sb, goneSlot, func(slot []byte) { + binary.LittleEndian.PutUint64(slot[172:180], uint64(time.Now().Add(-time.Hour).Unix())) + }) + journalAt := int64(sb.JournalRegionOffset + uint64(goneSlot)*sb.JournalSlotSize()) + header := journal.NewJournalSuperblock([16]byte{1}, 1, 1<<20) + header.State = journal.JournalStateDirty + damaged := header.Serialize() + damaged[100] ^= 0xFF + writeSector(t, dev, journalAt, damaged) + + c := checker() + plan, err := c.PlanRelease(goneSlot) + if err != nil { + t.Fatalf("PlanRelease of an expired slot with a damaged journal: %v", err) + } + if !strings.HasPrefix(plan.Slot.Journal, "damaged: ") || !strings.HasPrefix(plan.Discards, "the whole journal, unread") { + t.Errorf("plan reports journal %q, discards %q", plan.Slot.Journal, plan.Discards) + } + if err := c.Release(plan); err != nil { + t.Fatalf("Release: %v", err) + } + + slots, err := checker().Slots() + if err != nil { + t.Fatal(err) + } + if s := slots[goneSlot]; s.InUse() || !s.JournalFresh { + t.Errorf("after release slot %d is in use=%v with journal %q", goneSlot, s.InUse(), s.Journal) + } + replacement := node.NewRegistry(dev, sb.NodeRegistryOffset) + if got, err := replacement.Register(ctx, "replacement"); err != nil || got != goneSlot { + t.Fatalf("a new node registered in slot %d (%v), want released slot %d", got, err, goneSlot) + } + if result, err := journal.ScanAndRecover(dev, uint64(journalAt), sb.JournalSlotSize(), nil); result != nil || err != nil { + t.Errorf("mount recovery of the released journal = %+v, %v; want nothing to recover", result, err) + } +} + +// A node that registers between the plan and the release stops the release +// before it writes anything. +func TestReleaseSlotStopsWhenRegistryChanges(t *testing.T) { + t.Cleanup(fsck.SetHeartbeatWatch(2*node.HeartbeatInterval, 2*node.HeartbeatInterval)) + var sb *superblock.Superblock + dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) + ctx := context.Background() + + gone := node.NewRegistry(dev, sb.NodeRegistryOffset) + goneSlot, err := gone.Register(ctx, "gone") + if err != nil { + t.Fatal(err) + } + editSlot(t, dev, sb, goneSlot, func(slot []byte) { + binary.LittleEndian.PutUint64(slot[32:40], uint64(time.Now().Add(-time.Hour).Unix())) + }) + journalAt := int64(sb.JournalRegionOffset + uint64(goneSlot)*sb.JournalSlotSize()) + writeSector(t, dev, journalAt, []byte("not a journal")) + + c := fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) + plan, err := c.PlanRelease(goneSlot) + if err != nil { + t.Fatal(err) + } + if _, err := node.NewRegistry(dev, sb.NodeRegistryOffset).Register(ctx, "late"); err != nil { + t.Fatal(err) + } + if err := c.Release(plan); err == nil || !strings.Contains(err.Error(), "changed since fsck checked it") { + t.Fatalf("Release after a node registered: %v", err) + } + header := storage.NewAlignedBuffer(storage.SectorSize).Bytes() + if _, err := dev.ReadAt(header, journalAt); err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(header, []byte("not a journal")) { + t.Error("Release reset the journal although it stopped") + } +} + +func editSlot(t *testing.T, dev *storage.Device, sb *superblock.Superblock, slot int, edit func(slot []byte)) { + t.Helper() + off := int64(sb.NodeRegistryOffset) + int64(slot)*node.SlotSize + buf := storage.NewAlignedBuffer(node.SlotSize).Bytes() + if _, err := dev.ReadAt(buf, off); err != nil { + t.Fatal(err) + } + edit(buf) + binary.LittleEndian.PutUint32(buf[node.SlotSize-4:], storage.CRC32C(buf[:node.SlotSize-4])) + writeSector(t, dev, off, buf) +} diff --git a/internal/node/liveness.go b/internal/node/liveness.go index c60d58ed..9ec1c309 100644 --- a/internal/node/liveness.go +++ b/internal/node/liveness.go @@ -81,6 +81,19 @@ func (r *Registry) failed(slot *Slot, timeout time.Duration) bool { return false } +// LeaseExpiry returns when the slot's node stops counting as alive: its last +// heartbeat plus HeartbeatTimeout, or for a dead slot, the end of the recovery +// lease of the peer replaying its journal. An empty slot returns the zero time. +func (s *Slot) LeaseExpiry() time.Time { + switch s.State { + case NodeStateActive, NodeStateLeaving: + return time.Unix(s.HeartbeatTime, 0).Add(HeartbeatTimeout) + case NodeStateDead: + return time.Unix(s.RecoveryStart, 0).Add(recoveryLease) + } + return time.Time{} +} + // observe records a slot's heartbeat as failure detection sees it. Called with r.mu held. func (r *Registry) observe(slotIdx int, slot *Slot) { last, seen := r.sightings[slotIdx] diff --git a/internal/node/registry.go b/internal/node/registry.go index a5a8005f..f7fe3951 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -53,6 +53,20 @@ const ( NodeStateDead NodeState = 3 // Node failed (detected by others) ) +func (s NodeState) String() string { + switch s { + case NodeStateEmpty: + return "empty" + case NodeStateActive: + return "active" + case NodeStateLeaving: + return "leaving" + case NodeStateDead: + return "dead" + } + return fmt.Sprintf("unknown(%d)", uint32(s)) +} + // Slot represents a node's on-disk registration entry. // Size: 4096 bytes (sector-aligned for O_DIRECT atomic writes) type Slot struct { @@ -655,6 +669,11 @@ func serializeSlot(slot *Slot) []byte { } // deserializeSlot parses a Slot from bytes. +// ParseSlot decodes one raw registry slot. +func ParseSlot(buf []byte) (*Slot, error) { + return deserializeSlot(buf) +} + func deserializeSlot(buf []byte) (*Slot, error) { if len(buf) < SlotSize { return nil, ErrInvalidSlot @@ -876,6 +895,18 @@ func (r *Registry) clearFailedSlot(slotIdx int, uuid [16]byte) error { return err } +// ReleaseSlot empties slot slotIdx if it still holds expected, the raw slot +// bytes the caller checked, and reports whether it did. Only fsck calls it, +// for a slot whose node will not return. +func (r *Registry) ReleaseSlot(slotIdx int, expected []byte) (bool, error) { + if slotIdx < 0 || slotIdx >= MaxNodes { + return false, fmt.Errorf("slot index out of range: %d", slotIdx) + } + r.mu.Lock() + defer r.mu.Unlock() + return r.casSlot(slotIdx, expected, serializeSlot(&Slot{Magic: SlotMagic, State: NodeStateEmpty})) +} + // GetFailedNodes returns information about nodes that have failed (stale heartbeat). // Useful for manual inspection or triggering recovery outside the normal flow. func (r *Registry) GetFailedNodes() ([]*FailedNodeInfo, error) { diff --git a/tests/integration/fsck_test.go b/tests/integration/fsck_test.go index 68591fb4..4e50b140 100644 --- a/tests/integration/fsck_test.go +++ b/tests/integration/fsck_test.go @@ -3,18 +3,22 @@ package integration import ( "bytes" "context" + "encoding/binary" "errors" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "testing" + "time" "golang.org/x/sys/unix" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" "github.com/supporttools/GoClusterFS/internal/superblock" ) @@ -250,3 +254,68 @@ func TestFsckRepairsOlderSharing(t *testing.T) { t.Fatalf("problems after using the repaired image:\n%s", out) } } + +// TestFsckReleasesDeadSlot frees the slot of a node that will not return and +// whose journal header is damaged, so a mount can use the slot again. +func TestFsckReleasesDeadSlot(t *testing.T) { + cluster := NewMultiNodeCluster(t, 1, 50) + mfs := cluster.Node(0) + formatted := &syncBuffer{} + mfs.Log = formatted + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("format and start: %v", err) + } + slot := mustMatch(t, slotRe, formatted, "registration") + offset := mustMatch(t, journalInitRe, formatted, "journal init") + image := cluster.ImagePath + + expectFsck(t, 8, "mounted by node "+slot, "-release-slot", slot, image) + + if err := cluster.KillNode(0); err != nil { + t.Fatalf("kill: %v", err) + } + backdateHeartbeat(t, image, slot, time.Hour) + damageJournalHeader(t, image, offset) + + expectFsck(t, 0, "damaged: journal superblock checksum mismatch", "-list-slots", image) + expectFsck(t, 16, "-discard-journal needs -release-slot", "-discard-journal", image) + expectFsck(t, 0, "Nothing was written", "-release-slot", slot, image) + expectFsck(t, 0, "Released slot "+slot, "-release-slot", slot, "-discard-journal", image) + + restarted := &syncBuffer{} + mfs.Log = restarted + if err := cluster.StartNode(0, false); err != nil { + t.Fatalf("mount after releasing the slot: %v", err) + } + if got := mustMatch(t, journalInitRe, restarted, "journal init"); got != offset { + t.Errorf("mount started its journal at offset %s, want the released slot's offset %s", got, offset) + } +} + +func backdateHeartbeat(t *testing.T, imagePath, slot string, by time.Duration) { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatalf("open image: %v", err) + } + defer func() { _ = dev.Close() }() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatalf("read superblock: %v", err) + } + i, err := strconv.Atoi(slot) + if err != nil { + t.Fatalf("parse slot %q: %v", slot, err) + } + + off := int64(sb.NodeRegistryOffset) + int64(i)*node.SlotSize + buf := storage.NewAlignedBuffer(node.SlotSize).Bytes() + if _, err := dev.ReadAt(buf, off); err != nil { + t.Fatalf("read slot %d: %v", i, err) + } + binary.LittleEndian.PutUint64(buf[32:40], uint64(time.Now().Add(-by).Unix())) + binary.LittleEndian.PutUint32(buf[node.SlotSize-4:], storage.CRC32C(buf[:node.SlotSize-4])) + if _, err := dev.WriteAt(buf, off); err != nil { + t.Fatalf("write slot %d: %v", i, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 8bb1a635..9c44c9aa 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -837,6 +837,16 @@ scenarios: docker: TODO vm: TODO + - id: fsck-release-dead-slot + name: fsck releases a dead node's slot + description: -release-slot exits 8 while the node is mounted; after the node dies with a damaged journal header, -list-slots shows the damage, -release-slot prints what it discards without writing, -discard-journal frees the slot, and a mount then uses it + priority: high + # Docker/VM: their images do not ship goclusterfs-fsck. The Go test runs both binaries against a real mount. + environments: + go: fsck_test.go::TestFsckReleasesDeadSlot + docker: TODO + vm: TODO + - id: fsck-runtime-image-unit name: fsck accepts everything the runtime writes description: Snapshots, clones, snapshot clones, copies on write in inline and tree files, deletes and unlinks leave no fsck problem @@ -872,6 +882,20 @@ scenarios: environments: go: internal/fsck/layout_test.go::TestFsckRepairRefusesInUseFilesystem + - id: fsck-release-slot + name: fsck slot release checks + description: Release refuses while any node runs or the slot's heartbeat or recovery lease is recent; an expired slot with a damaged journal is released, its journal reads as never initialized, and a new node registers in it + priority: go_only + environments: + go: internal/fsck/slots_test.go::TestReleaseSlot + + - id: fsck-release-slot-registry-changes + name: fsck slot release stops when the registry changes + description: A node that registers between the release check and the write stops the release before the journal is reset + priority: go_only + environments: + go: internal/fsck/slots_test.go::TestReleaseSlotStopsWhenRegistryChanges + - id: fsck-bad-layout name: fsck stops on a layout it cannot trust description: A zero inode size, no or too many data blocks, a wrapping offset or overlapping regions abort fsck in phase 1 with exit code 8 From f0410ac43e185f6a0bdb61248fc89558708d7f05 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:11:00 -0500 Subject: [PATCH 2/3] fix(fsck): take the recovery lease before releasing a slot Review fixes for slot release: - A node mounting between the registry check and the writes could get its new journal header wiped. Release now takes the slot's recovery lease first, as a peer replaying the slot would, so Register and failure detection skip the slot while fsck resets its journal. - Release requires a superblock that mount accepts, since it writes at offsets the superblock gives. - A journal a mount could replay in full is refused without -force. - The discard count includes transactions read before a bad record. - A slot number the registry does not have exits 16; a release exits 1. Co-Authored-By: Claude Opus 5.5 --- cmd/fsck/main.go | 17 ++- cmd/fsck/slots.go | 24 ++-- internal/fsck/export_test.go | 8 ++ internal/fsck/slots.go | 93 ++++++++++----- internal/fsck/slots_test.go | 200 ++++++++++++++++++++++++--------- internal/node/registry.go | 40 ++++++- tests/integration/fsck_test.go | 3 +- 7 files changed, 281 insertions(+), 104 deletions(-) diff --git a/cmd/fsck/main.go b/cmd/fsck/main.go index ba9fe12b..6a535407 100644 --- a/cmd/fsck/main.go +++ b/cmd/fsck/main.go @@ -21,13 +21,14 @@ // Exit codes, bits as with fsck(8): // // 0: No errors (warnings may remain) -// 1: Errors were repaired +// 1: Errors were repaired, or -release-slot -discard-journal freed a slot // 4: Errors remain on disk // 8: I/O failure, or the check could not finish or was refused // 16: Usage error package main import ( + "errors" "flag" "fmt" "os" @@ -41,7 +42,7 @@ func main() { var ( repair = flag.Bool("repair", false, "Repair problems automatically where possible (see Repair below)") dryRun = flag.Bool("dry-run", false, "With -repair, report what would change without writing") - force = flag.Bool("force", false, "Keep checking after a fatal error; with -repair, repair even if a stopped node's journal still needs replay") + force = flag.Bool("force", false, "Keep checking after a fatal error; with -repair, repair even if a stopped node's journal still needs replay; with -release-slot, discard a journal a mount could replay") verbose = flag.Bool("verbose", false, "Enable verbose progress output") quiet = flag.Bool("quiet", false, "Only print errors and summary") stopOnFatal = flag.Bool("stop-on-fatal", true, "Stop checking when a fatal error is found") @@ -71,7 +72,7 @@ func main() { fmt.Fprint(os.Stderr, slotsHelp) fmt.Fprintf(os.Stderr, "\nExit codes (bits, as with fsck(8); 5 means some errors were repaired and some remain):\n") fmt.Fprintf(os.Stderr, " 0: No errors (warnings may remain)\n") - fmt.Fprintf(os.Stderr, " 1: Errors were repaired\n") + fmt.Fprintf(os.Stderr, " 1: Errors were repaired, or -release-slot -discard-journal freed a slot\n") fmt.Fprintf(os.Stderr, " 4: Errors remain on disk\n") fmt.Fprintf(os.Stderr, " 8: I/O failure, or the check could not finish or was refused\n") fmt.Fprintf(os.Stderr, " 16: Usage error\n") @@ -138,11 +139,17 @@ func main() { if *listSlots { err = printSlots(os.Stdout, checker) } else { - err = releaseSlot(os.Stdout, checker, *releaseSlotN, *discardJournal) + err = releaseSlot(os.Stdout, checker, *releaseSlotN, *discardJournal, *force) } - if err != nil { + switch { + case errors.Is(err, fsck.ErrNoSuchSlot): + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(fsck.ExitUsage) + case err != nil: fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(fsck.ExitOperational) + case *discardJournal: + os.Exit(fsck.ExitCorrected) } os.Exit(fsck.ExitClean) } diff --git a/cmd/fsck/slots.go b/cmd/fsck/slots.go index af2759ce..762a2c77 100644 --- a/cmd/fsck/slots.go +++ b/cmd/fsck/slots.go @@ -13,16 +13,18 @@ import ( const slotsHelp = ` Node registry slots: - Every mounted node holds a registry slot with its own journal. A crashed - node's slot frees once another node replays its journal, or when the same - host mounts again. -list-slots shows each slot that is in use or whose - journal needs attention. + Every mounted node holds a registry slot with its own journal. When a node + crashes, a peer replays its journal and frees the slot, or a mount on any + host takes the slot and replays the journal first. A slot whose journal + cannot be replayed, for example because its header is damaged, stays + taken. -list-slots shows each slot that is in use or whose journal needs + attention. - -release-slot N frees the slot of a node that will not return, for example - one whose journal header is damaged. It refuses while any node has the - filesystem mounted, or until a minute after the slot's lease ended, and - prints what releasing would discard. Add -discard-journal to reset the - slot's journal and free the slot. + -release-slot N checks that slot N can be freed and prints what freeing it + would discard. It refuses while any node has the filesystem mounted, until + a minute after the slot's lease ended, and, without -force, while a mount + could still replay the journal. Add -discard-journal to reset the slot's + journal and free the slot; fsck then exits 1. ` // printSlots lists the slots that are in use or whose journal was written @@ -57,8 +59,8 @@ func printSlots(out io.Writer, checker *fsck.Checker) error { // releaseSlot checks slot i and prints what releasing it discards. With // discard set it also releases the slot. -func releaseSlot(out io.Writer, checker *fsck.Checker, i int, discard bool) error { - plan, err := checker.PlanRelease(i) +func releaseSlot(out io.Writer, checker *fsck.Checker, i int, discard, force bool) error { + plan, err := checker.PlanRelease(i, force) if err != nil { return err } diff --git a/internal/fsck/export_test.go b/internal/fsck/export_test.go index 266383a9..5a35983b 100644 --- a/internal/fsck/export_test.go +++ b/internal/fsck/export_test.go @@ -16,3 +16,11 @@ func SetBeforeRepairWrites(f func()) (restore func()) { beforeRepairWrites = f return func() { beforeRepairWrites = old } } + +// SetReleaseHooks runs before once Release has checked the registry, and +// during once Release holds the slot's recovery lease. +func SetReleaseHooks(before, during func()) (restore func()) { + oldBefore, oldDuring := beforeReleaseWrites, whileReleaseHoldsLease + beforeReleaseWrites, whileReleaseHoldsLease = before, during + return func() { beforeReleaseWrites, whileReleaseHoldsLease = oldBefore, oldDuring } +} diff --git a/internal/fsck/slots.go b/internal/fsck/slots.go index bfb9118c..3e4f9543 100644 --- a/internal/fsck/slots.go +++ b/internal/fsck/slots.go @@ -9,12 +9,20 @@ import ( "github.com/supporttools/GoClusterFS/internal/journal" "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" ) // ReleaseMargin is how long a slot's lease must have been over before fsck // releases the slot, so a node that is only slow, or whose clock lags, keeps it. const ReleaseMargin = time.Minute +// ErrNoSuchSlot means a release named a slot the registry does not have. +var ErrNoSuchSlot = errors.New("no such node registry slot") + +// Test hooks: beforeReleaseWrites runs after Release checks the registry, and +// whileReleaseHoldsLease runs once Release holds the slot's recovery lease. +var beforeReleaseWrites, whileReleaseHoldsLease func() + // SlotStatus describes a node registry slot and its journal. type SlotStatus struct { Index int @@ -71,15 +79,20 @@ type ReleasePlan struct { Discards string // what resetting the slot's journal throws away } -// PlanRelease checks that slot i can be released: no node runs, the slot is -// readable, and its lease ended ReleaseMargin ago. It writes nothing. -func (c *Checker) PlanRelease(i int) (*ReleasePlan, error) { +// PlanRelease checks that slot i can be released: mount would accept the +// superblock, no node runs, the slot is readable, its lease ended +// ReleaseMargin ago, and, unless force is set, a mount could not replay its +// journal instead. It writes nothing. +func (c *Checker) PlanRelease(i int, force bool) (*ReleasePlan, error) { + if _, err := superblock.Read(c.dev); err != nil { + return nil, fmt.Errorf("release refused: mount would reject the superblock (%v); run fsck first", err) + } slots, err := c.Slots() if err != nil { return nil, err } if i < 0 || i >= len(slots) { - return nil, fmt.Errorf("slot %d does not exist; slots are 0 to %d", i, len(slots)-1) + return nil, fmt.Errorf("%w: slot %d; slots are 0 to %d", ErrNoSuchSlot, i, len(slots)-1) } for _, s := range slots { if s.Running { @@ -94,12 +107,17 @@ func (c *Checker) PlanRelease(i int) (*ReleasePlan, error) { return nil, fmt.Errorf("release refused: %s may still be alive; run again after %s", s.Name(), after.Format(time.DateTime)) } - return &ReleasePlan{Slot: s, Discards: discards(c.slotJournal(i))}, nil + discards, replayable := journalContents(c.slotJournal(i)) + if replayable && !force { + return nil, fmt.Errorf("release refused: slot %d's journal holds %s, which a mount can replay; "+ + "mount a node to replay it, or add -force to discard it", i, discards) + } + return &ReleasePlan{Slot: s, Discards: discards}, nil } -// Release resets the plan's journal and empties its registry slot, so a node -// can register there with a new journal. It stops if any registry slot -// changed since PlanRelease watched the registry. +// Release takes the plan's slot under a recovery lease, resets its journal +// and empties the slot, so a node can register there with a new journal. It +// writes nothing if any registry slot changed since PlanRelease watched it. func (c *Checker) Release(plan *ReleasePlan) error { i := plan.Slot.Index now, err := c.readRegistry() @@ -107,22 +125,32 @@ func (c *Checker) Release(plan *ReleasePlan) error { return fmt.Errorf("read the node registry: %w", err) } if changed := changedSlots(c.registry, now); len(changed) > 0 { - return fmt.Errorf("release stopped: node registry slot %d changed since fsck checked it; run again", changed[0]) + return fmt.Errorf("release stopped: node registry slot %d changed since fsck checked it; nothing was written", changed[0]) } - - offset, _ := c.journalArea(i) - if _, err := c.dev.WriteAt(storage.NewAlignedBuffer(storage.SectorSize).Bytes(), int64(offset)); err != nil { - return fmt.Errorf("reset the journal of slot %d: %w", i, err) + if beforeReleaseWrites != nil { + beforeReleaseWrites() } - if err := c.dev.Sync(); err != nil { - return fmt.Errorf("sync: %w", err) + + reset := false + resetJournal := func() error { + if whileReleaseHoldsLease != nil { + whileReleaseHoldsLease() + } + offset, _ := c.journalArea(i) + if _, err := c.dev.WriteAt(storage.NewAlignedBuffer(storage.SectorSize).Bytes(), int64(offset)); err != nil { + return fmt.Errorf("reset the journal: %w", err) + } + reset = true + return c.dev.Sync() } - released, err := node.NewRegistry(c.dev, c.sbData.NodeRegistryOffset).ReleaseSlot(i, plan.Slot.raw) - if err != nil { + released, err := node.NewRegistry(c.dev, c.sbData.NodeRegistryOffset).ReleaseSlot(i, plan.Slot.raw, resetJournal) + switch { + case err != nil && reset: + return fmt.Errorf("release slot %d: %w; its journal is reset but the slot is not free, run again", i, err) + case err != nil: return fmt.Errorf("release slot %d: %w", i, err) - } - if !released { - return fmt.Errorf("slot %d changed while fsck released it: its journal is reset but the slot is not; run again", i) + case !released: + return fmt.Errorf("release stopped: another node took slot %d since fsck checked it; its journal was not touched", i) } return c.dev.Sync() } @@ -150,24 +178,29 @@ func journalState(region *journal.Region, err error) string { return strings.ToLower(region.State().String()) } -// discards says what resetting a journal throws away. -func discards(region *journal.Region, err error) string { +// journalContents says what resetting a journal throws away, and whether a +// mount could replay all of it instead. +func journalContents(region *journal.Region, err error) (discards string, replayable bool) { switch { case errors.Is(err, journal.ErrJournalUninitialized): - return "nothing: the journal was never initialized" + return "nothing: the journal was never initialized", false case err != nil: - return "the whole journal, unread: " + rootCause(err).Error() + return "the whole journal, unread: " + rootCause(err).Error(), false case region.State() == journal.JournalStateClean: - return "nothing: the journal is clean" + return "nothing: the journal is clean", false case !region.NeedsRecovery(): - return "the whole journal, unread: its state is " + strings.ToLower(region.State().String()) + return "the whole journal, unread: its state is " + strings.ToLower(region.State().String()), false + } + _, stats, err := journal.DryRunRecover(region) + pending := stats.TxnCommitted - stats.IntentsSkipped + discards = fmt.Sprintf("%d committed transactions that were never replayed", pending) + if pending == 1 { + discards = "1 committed transaction that was never replayed" } - txns, _, err := journal.DryRunRecover(region) - what := fmt.Sprintf("%d committed transactions that were never replayed", len(txns)) if err != nil { - what += fmt.Sprintf(", and whatever follows a record fsck cannot read (%v)", err) + return discards + fmt.Sprintf(", and whatever follows a record fsck cannot read (%v)", err), false } - return what + return discards, pending > 0 } func rootCause(err error) error { diff --git a/internal/fsck/slots_test.go b/internal/fsck/slots_test.go index d404bd19..3046ab81 100644 --- a/internal/fsck/slots_test.go +++ b/internal/fsck/slots_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "errors" "strings" "testing" "time" @@ -20,16 +21,15 @@ func TestReleaseSlot(t *testing.T) { var sb *superblock.Superblock dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) ctx := context.Background() - checker := func() *fsck.Checker { return fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) } refused := func(slot int, reason string) { t.Helper() - if _, err := checker().PlanRelease(slot); err == nil || !strings.Contains(err.Error(), reason) { + if _, err := newChecker(dev).PlanRelease(slot, false); err == nil || !strings.Contains(err.Error(), reason) { t.Fatalf("PlanRelease(%d) error = %v, want one containing %q", slot, err, reason) } } gone := node.NewRegistry(dev, sb.NodeRegistryOffset) - goneSlot, err := gone.Register(ctx, "gone") + slot, err := gone.Register(ctx, "gone") if err != nil { t.Fatal(err) } @@ -44,30 +44,26 @@ func TestReleaseSlot(t *testing.T) { t.Fatal(err) } gone.Stop() - refused(goneSlot, "mounted by node 1 (live)") + refused(slot, "mounted by node 1 (live)") if err := live.Unregister(ctx); err != nil { t.Fatal(err) } - refused(goneSlot, "node 0 (gone) may still be alive") + refused(slot, "node 0 (gone) may still be alive") + setHeartbeat(t, dev, sb, slot, time.Now().Add(-30*time.Second)) + refused(slot, "may still be alive") - editSlot(t, dev, sb, goneSlot, func(slot []byte) { - binary.LittleEndian.PutUint32(slot[4:8], uint32(node.NodeStateDead)) - binary.LittleEndian.PutUint64(slot[172:180], uint64(time.Now().Unix())) + editSlot(t, dev, sb, slot, func(s []byte) { + binary.LittleEndian.PutUint32(s[4:8], uint32(node.NodeStateDead)) + binary.LittleEndian.PutUint64(s[172:180], uint64(time.Now().Unix())) }) - refused(goneSlot, "may still be alive") - - editSlot(t, dev, sb, goneSlot, func(slot []byte) { - binary.LittleEndian.PutUint64(slot[172:180], uint64(time.Now().Add(-time.Hour).Unix())) + refused(slot, "may still be alive") + editSlot(t, dev, sb, slot, func(s []byte) { + binary.LittleEndian.PutUint64(s[172:180], uint64(time.Now().Add(-time.Hour).Unix())) }) - journalAt := int64(sb.JournalRegionOffset + uint64(goneSlot)*sb.JournalSlotSize()) - header := journal.NewJournalSuperblock([16]byte{1}, 1, 1<<20) - header.State = journal.JournalStateDirty - damaged := header.Serialize() - damaged[100] ^= 0xFF - writeSector(t, dev, journalAt, damaged) + journalAt := damageJournal(t, dev, sb, slot) - c := checker() - plan, err := c.PlanRelease(goneSlot) + c := newChecker(dev) + plan, err := c.PlanRelease(slot, false) if err != nil { t.Fatalf("PlanRelease of an expired slot with a damaged journal: %v", err) } @@ -78,69 +74,171 @@ func TestReleaseSlot(t *testing.T) { t.Fatalf("Release: %v", err) } - slots, err := checker().Slots() + slots, err := newChecker(dev).Slots() if err != nil { t.Fatal(err) } - if s := slots[goneSlot]; s.InUse() || !s.JournalFresh { - t.Errorf("after release slot %d is in use=%v with journal %q", goneSlot, s.InUse(), s.Journal) + if s := slots[slot]; s.InUse() || !s.JournalFresh { + t.Errorf("after release slot %d is in use=%v with journal %q", slot, s.InUse(), s.Journal) } replacement := node.NewRegistry(dev, sb.NodeRegistryOffset) - if got, err := replacement.Register(ctx, "replacement"); err != nil || got != goneSlot { - t.Fatalf("a new node registered in slot %d (%v), want released slot %d", got, err, goneSlot) + if got, err := replacement.Register(ctx, "replacement"); err != nil || got != slot { + t.Fatalf("a new node registered in slot %d (%v), want released slot %d", got, err, slot) } if result, err := journal.ScanAndRecover(dev, uint64(journalAt), sb.JournalSlotSize(), nil); result != nil || err != nil { t.Errorf("mount recovery of the released journal = %+v, %v; want nothing to recover", result, err) } } -// A node that registers between the plan and the release stops the release -// before it writes anything. -func TestReleaseSlotStopsWhenRegistryChanges(t *testing.T) { +func TestReleaseSlotRefusals(t *testing.T) { t.Cleanup(fsck.SetHeartbeatWatch(2*node.HeartbeatInterval, 2*node.HeartbeatInterval)) var sb *superblock.Superblock dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) - ctx := context.Background() + slot := goneNode(t, dev, sb) - gone := node.NewRegistry(dev, sb.NodeRegistryOffset) - goneSlot, err := gone.Register(ctx, "gone") - if err != nil { - t.Fatal(err) + if _, err := newChecker(dev).PlanRelease(node.MaxNodes, false); !errors.Is(err, fsck.ErrNoSuchSlot) { + t.Errorf("PlanRelease of slot %d: %v, want %v", node.MaxNodes, err, fsck.ErrNoSuchSlot) } - editSlot(t, dev, sb, goneSlot, func(slot []byte) { - binary.LittleEndian.PutUint64(slot[32:40], uint64(time.Now().Add(-time.Hour).Unix())) - }) - journalAt := int64(sb.JournalRegionOffset + uint64(goneSlot)*sb.JournalSlotSize()) - writeSector(t, dev, journalAt, []byte("not a journal")) - c := fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) - plan, err := c.PlanRelease(goneSlot) + j, err := journal.Init(dev, sb.JournalRegionOffset+uint64(slot)*sb.JournalSlotSize(), sb.JournalSlotSize(), + [16]byte{1}, 1, journal.DefaultConfig()) if err != nil { t.Fatal(err) } - if _, err := node.NewRegistry(dev, sb.NodeRegistryOffset).Register(ctx, "late"); err != nil { + txn := j.Begin() + if err := txn.LogInodeDelete(&journal.InodeDeletePayload{InodeNum: 7}); err != nil { + t.Fatal(err) + } + if _, err := j.Commit(txn); err != nil { t.Fatal(err) } - if err := c.Release(plan); err == nil || !strings.Contains(err.Error(), "changed since fsck checked it") { - t.Fatalf("Release after a node registered: %v", err) + if _, err := newChecker(dev).PlanRelease(slot, false); err == nil || !strings.Contains(err.Error(), "mount a node to replay it") { + t.Errorf("PlanRelease of a replayable journal: %v", err) + } + if plan, err := newChecker(dev).PlanRelease(slot, true); err != nil || plan.Discards != "1 committed transaction that was never replayed" { + t.Errorf("PlanRelease with force: plan %+v, error %v", plan, err) + } + + slotAt := int64(sb.NodeRegistryOffset) + int64(slot)*node.SlotSize + corrupt := readSector(t, dev, slotAt) + corrupt[100] ^= 0xFF + writeSector(t, dev, slotAt, corrupt) + if _, err := newChecker(dev).PlanRelease(slot, true); err == nil || !strings.Contains(err.Error(), "unreadable") { + t.Errorf("PlanRelease of an unreadable slot: %v", err) + } + + writeSector(t, dev, 0, []byte("not a superblock")) + if _, err := newChecker(dev).PlanRelease(slot, true); err == nil || !strings.Contains(err.Error(), "mount would reject the superblock") { + t.Errorf("PlanRelease with a bad superblock: %v", err) + } +} + +// A node that takes the slot between the registry check and the lease stops +// the release before the journal is touched. Once fsck holds the lease, a +// mounting node takes another slot. +func TestReleaseSlotLease(t *testing.T) { + t.Cleanup(fsck.SetHeartbeatWatch(2*node.HeartbeatInterval, 2*node.HeartbeatInterval)) + var sb *superblock.Superblock + dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) + ctx := context.Background() + release := func(slot int) error { + t.Helper() + c := newChecker(dev) + plan, err := c.PlanRelease(slot, false) + if err != nil { + t.Fatal(err) + } + return c.Release(plan) + } + + slot := goneNode(t, dev, sb) + journalAt := damageJournal(t, dev, sb, slot) + damaged := readSector(t, dev, journalAt) + var late *node.Registry + t.Cleanup(fsck.SetReleaseHooks(func() { + late = node.NewRegistry(dev, sb.NodeRegistryOffset) + if got, err := late.Register(ctx, "late"); err != nil || got != slot { + t.Errorf("late node took slot %d (%v), want the stale slot %d", got, err, slot) + } + }, nil)) + if err := release(slot); err == nil || !strings.Contains(err.Error(), "its journal was not touched") { + t.Fatalf("Release after a node took the slot: %v", err) + } + if !bytes.Equal(readSector(t, dev, journalAt), damaged) { + t.Error("Release reset the journal of a slot another node took") } - header := storage.NewAlignedBuffer(storage.SectorSize).Bytes() - if _, err := dev.ReadAt(header, journalAt); err != nil { + if err := late.Unregister(ctx); err != nil { t.Fatal(err) } - if !bytes.HasPrefix(header, []byte("not a journal")) { - t.Error("Release reset the journal although it stopped") + + slot = goneNode(t, dev, sb) + damageJournal(t, dev, sb, slot) + t.Cleanup(fsck.SetReleaseHooks(nil, func() { + if got, err := node.NewRegistry(dev, sb.NodeRegistryOffset).Register(ctx, "during"); err != nil || got == slot { + t.Errorf("node mounting during the release took slot %d (%v)", got, err) + } + })) + if err := release(slot); err != nil { + t.Fatalf("Release while a node mounts: %v", err) + } + + slot = goneNode(t, dev, sb) + damageJournal(t, dev, sb, slot) + t.Cleanup(fsck.SetReleaseHooks(nil, func() { + setHeartbeat(t, dev, sb, slot, time.Now()) + })) + if err := release(slot); err == nil || !strings.Contains(err.Error(), "its journal is reset but the slot is not free") { + t.Fatalf("Release of a slot written during the lease: %v", err) } } -func editSlot(t *testing.T, dev *storage.Device, sb *superblock.Superblock, slot int, edit func(slot []byte)) { +func newChecker(dev *storage.Device) *fsck.Checker { + return fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) +} + +// goneNode registers a node that stopped an hour ago and returns its slot. +func goneNode(t *testing.T, dev *storage.Device, sb *superblock.Superblock) int { t.Helper() - off := int64(sb.NodeRegistryOffset) + int64(slot)*node.SlotSize - buf := storage.NewAlignedBuffer(node.SlotSize).Bytes() - if _, err := dev.ReadAt(buf, off); err != nil { + slot, err := node.NewRegistry(dev, sb.NodeRegistryOffset).Register(context.Background(), "gone") + if err != nil { t.Fatal(err) } + setHeartbeat(t, dev, sb, slot, time.Now().Add(-time.Hour)) + return slot +} + +// damageJournal writes a dirty journal header with a bad checksum into slot's +// journal and returns the journal's offset. +func damageJournal(t *testing.T, dev *storage.Device, sb *superblock.Superblock, slot int) int64 { + t.Helper() + at := int64(sb.JournalRegionOffset + uint64(slot)*sb.JournalSlotSize()) + header := journal.NewJournalSuperblock([16]byte{1}, 1, 1<<20) + header.State = journal.JournalStateDirty + damaged := header.Serialize() + damaged[100] ^= 0xFF + writeSector(t, dev, at, damaged) + return at +} + +func setHeartbeat(t *testing.T, dev *storage.Device, sb *superblock.Superblock, slot int, at time.Time) { + t.Helper() + editSlot(t, dev, sb, slot, func(s []byte) { binary.LittleEndian.PutUint64(s[32:40], uint64(at.Unix())) }) +} + +func editSlot(t *testing.T, dev *storage.Device, sb *superblock.Superblock, slot int, edit func(slot []byte)) { + t.Helper() + off := int64(sb.NodeRegistryOffset) + int64(slot)*node.SlotSize + buf := readSector(t, dev, off) edit(buf) binary.LittleEndian.PutUint32(buf[node.SlotSize-4:], storage.CRC32C(buf[:node.SlotSize-4])) writeSector(t, dev, off, buf) } + +func readSector(t *testing.T, dev *storage.Device, offset int64) []byte { + t.Helper() + buf := storage.NewAlignedBuffer(storage.SectorSize).Bytes() + if _, err := dev.ReadAt(buf, offset); err != nil { + t.Fatal(err) + } + return buf +} diff --git a/internal/node/registry.go b/internal/node/registry.go index f7fe3951..6a0726ca 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -668,12 +668,12 @@ func serializeSlot(slot *Slot) []byte { return buf } -// deserializeSlot parses a Slot from bytes. // ParseSlot decodes one raw registry slot. func ParseSlot(buf []byte) (*Slot, error) { return deserializeSlot(buf) } +// deserializeSlot parses a Slot from bytes. func deserializeSlot(buf []byte) (*Slot, error) { if len(buf) < SlotSize { return nil, ErrInvalidSlot @@ -895,16 +895,44 @@ func (r *Registry) clearFailedSlot(slotIdx int, uuid [16]byte) error { return err } -// ReleaseSlot empties slot slotIdx if it still holds expected, the raw slot -// bytes the caller checked, and reports whether it did. Only fsck calls it, -// for a slot whose node will not return. -func (r *Registry) ReleaseSlot(slotIdx int, expected []byte) (bool, error) { +// ReleaseSlot frees slot slotIdx, whose node will not return, if the slot +// still holds expected, the raw bytes the caller checked. Like a peer that +// replays a crashed node's journal, it first takes the slot's recovery lease, +// so neither Register nor failure detection touches the slot while reset +// clears its journal; then it empties the slot. It returns false without +// running reset if the slot changed or another node won the lease. +func (r *Registry) ReleaseSlot(slotIdx int, expected []byte, reset func() error) (bool, error) { if slotIdx < 0 || slotIdx >= MaxNodes { return false, fmt.Errorf("slot index out of range: %d", slotIdx) } r.mu.Lock() defer r.mu.Unlock() - return r.casSlot(slotIdx, expected, serializeSlot(&Slot{Magic: SlotMagic, State: NodeStateEmpty})) + + slot, err := deserializeSlot(expected) + if err != nil { + return false, err + } + slot.Magic = SlotMagic + slot.State = NodeStateDead + slot.RecoveryOwner = r.localUUID + slot.RecoveryStart = time.Now().Unix() + leased := serializeSlot(slot) + if ok, err := r.casSlot(slotIdx, expected, leased); err != nil || !ok { + return false, err + } + time.Sleep(claimSettle) + if current, _, err := r.readSlotRaw(slotIdx); err != nil || !bytes.Equal(current, leased) { + return false, err + } + + if err := reset(); err != nil { + return false, err + } + ok, err := r.casSlot(slotIdx, leased, serializeSlot(&Slot{Magic: SlotMagic, State: NodeStateEmpty})) + if err == nil && !ok { + err = fmt.Errorf("slot %d changed while its recovery lease was held", slotIdx) + } + return ok, err } // GetFailedNodes returns information about nodes that have failed (stale heartbeat). diff --git a/tests/integration/fsck_test.go b/tests/integration/fsck_test.go index 4e50b140..cd35fbb8 100644 --- a/tests/integration/fsck_test.go +++ b/tests/integration/fsck_test.go @@ -279,8 +279,9 @@ func TestFsckReleasesDeadSlot(t *testing.T) { expectFsck(t, 0, "damaged: journal superblock checksum mismatch", "-list-slots", image) expectFsck(t, 16, "-discard-journal needs -release-slot", "-discard-journal", image) + expectFsck(t, 16, "no such node registry slot", "-release-slot", "64", image) expectFsck(t, 0, "Nothing was written", "-release-slot", slot, image) - expectFsck(t, 0, "Released slot "+slot, "-release-slot", slot, "-discard-journal", image) + expectFsck(t, 1, "Released slot "+slot, "-release-slot", slot, "-discard-journal", image) restarted := &syncBuffer{} mfs.Log = restarted From 5e6459db3662ff4c75e31f569110a0729dd71225 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:30:27 -0500 Subject: [PATCH 3/3] fix(fsck): bound the release lease and require -force when a journal is unreadable Review follow-ups for slot release: - ReleaseSlot leaves the slot dead under its lease instead of freeing it when the journal reset took more than half the recovery lease. - A dry run that cannot read the whole journal now needs -force, since a mount may still replay it. - Tests for fsck crashing while it holds the lease, before and after the journal reset, and for a slow reset. - The manifest names the slot release tests that exist. Co-Authored-By: Claude Opus 5.5 --- internal/fsck/slots.go | 32 +++++++------ internal/fsck/slots_test.go | 84 ++++++++++++++++++++++++++++++++++ internal/node/registry.go | 11 ++++- internal/node/registry_test.go | 27 +++++++++++ tests/test-manifest.yaml | 29 ++++++++++-- 5 files changed, 164 insertions(+), 19 deletions(-) diff --git a/internal/fsck/slots.go b/internal/fsck/slots.go index 3e4f9543..538df3ff 100644 --- a/internal/fsck/slots.go +++ b/internal/fsck/slots.go @@ -107,10 +107,10 @@ func (c *Checker) PlanRelease(i int, force bool) (*ReleasePlan, error) { return nil, fmt.Errorf("release refused: %s may still be alive; run again after %s", s.Name(), after.Format(time.DateTime)) } - discards, replayable := journalContents(c.slotJournal(i)) - if replayable && !force { - return nil, fmt.Errorf("release refused: slot %d's journal holds %s, which a mount can replay; "+ - "mount a node to replay it, or add -force to discard it", i, discards) + discards, keep := journalContents(c.slotJournal(i)) + if keep != "" && !force { + return nil, fmt.Errorf("release refused: slot %d's journal holds %s, and %s; "+ + "mount a node to replay it, or add -force to discard it", i, discards, keep) } return &ReleasePlan{Slot: s, Discards: discards}, nil } @@ -178,18 +178,18 @@ func journalState(region *journal.Region, err error) string { return strings.ToLower(region.State().String()) } -// journalContents says what resetting a journal throws away, and whether a -// mount could replay all of it instead. -func journalContents(region *journal.Region, err error) (discards string, replayable bool) { +// journalContents says what resetting a journal throws away and, when a mount +// might replay it instead, why. +func journalContents(region *journal.Region, err error) (discards, keep string) { switch { case errors.Is(err, journal.ErrJournalUninitialized): - return "nothing: the journal was never initialized", false + return "nothing: the journal was never initialized", "" case err != nil: - return "the whole journal, unread: " + rootCause(err).Error(), false + return "the whole journal, unread: " + rootCause(err).Error(), "" case region.State() == journal.JournalStateClean: - return "nothing: the journal is clean", false + return "nothing: the journal is clean", "" case !region.NeedsRecovery(): - return "the whole journal, unread: its state is " + strings.ToLower(region.State().String()), false + return "the whole journal, unread: its state is " + strings.ToLower(region.State().String()), "" } _, stats, err := journal.DryRunRecover(region) pending := stats.TxnCommitted - stats.IntentsSkipped @@ -197,10 +197,14 @@ func journalContents(region *journal.Region, err error) (discards string, replay if pending == 1 { discards = "1 committed transaction that was never replayed" } - if err != nil { - return discards + fmt.Sprintf(", and whatever follows a record fsck cannot read (%v)", err), false + switch { + case err != nil: + return discards + ", and whatever follows a record fsck cannot read", + fmt.Sprintf("fsck could not read all of it (%v), so a mount may still replay it", err) + case pending > 0: + return discards, "a mount can replay it" } - return discards, pending > 0 + return discards, "" } func rootCause(err error) error { diff --git a/internal/fsck/slots_test.go b/internal/fsck/slots_test.go index 3046ab81..9321f706 100644 --- a/internal/fsck/slots_test.go +++ b/internal/fsck/slots_test.go @@ -192,6 +192,90 @@ func TestReleaseSlotLease(t *testing.T) { } } +// fsck can crash after it takes a slot's recovery lease, before or after it +// resets the journal. The lease keeps mounts and peers away until it lapses; +// then a peer replays the journal it finds: it fails on the untouched damaged +// one, and frees the slot after the reset one. +func TestReleaseSlotCrash(t *testing.T) { + for _, tt := range []struct { + name string + resetFirst bool + }{{"before reset", false}, {"after reset", true}} { + t.Run(tt.name, func(t *testing.T) { + var sb *superblock.Superblock + dev := formattedDevice(t, func(s *superblock.Superblock) { sb = s }) + ctx := context.Background() + gone := node.NewRegistry(dev, sb.NodeRegistryOffset) + gone.SetJournalArea(sb.JournalRegionOffset, sb.JournalSlotSize()) + slot, err := gone.Register(ctx, "gone") + if err != nil { + t.Fatal(err) + } + setHeartbeat(t, dev, sb, slot, time.Now().Add(-time.Hour)) + journalAt := damageJournal(t, dev, sb, slot) + + crash := errors.New("fsck crashed") + raw := readSector(t, dev, int64(sb.NodeRegistryOffset)+int64(slot)*node.SlotSize) + released, err := node.NewRegistry(dev, sb.NodeRegistryOffset).ReleaseSlot(slot, raw, func() error { + if tt.resetFirst { + writeSector(t, dev, journalAt, nil) + } + return crash + }) + if released || !errors.Is(err, crash) { + t.Fatalf("ReleaseSlot with a crashing reset = %v, %v", released, err) + } + journalLeft := readSector(t, dev, journalAt) + + peer := node.NewRegistry(dev, sb.NodeRegistryOffset) + peer.SetJournalArea(sb.JournalRegionOffset, sb.JournalSlotSize()) + if got, err := peer.Register(ctx, "peer"); err != nil || got == slot { + t.Fatalf("a node mounting during the lease took slot %d (%v)", got, err) + } + if replays := detectFailures(peer, dev); len(replays) != 0 { + t.Fatalf("a peer replayed slot %d during the lease", slot) + } + + editSlot(t, dev, sb, slot, func(s []byte) { + binary.LittleEndian.PutUint64(s[172:180], uint64(time.Now().Add(-61*time.Second).Unix())) + }) + replays := detectFailures(peer, dev) + if len(replays) != 1 || replays[0].slot != slot { + t.Fatalf("after the lease lapsed the peer replayed %+v, want slot %d once", replays, slot) + } + state, err := node.ParseSlot(readSector(t, dev, int64(sb.NodeRegistryOffset)+int64(slot)*node.SlotSize)) + if err != nil { + t.Fatal(err) + } + switch { + case tt.resetFirst && (replays[0].err != nil || state.State != node.NodeStateEmpty): + t.Errorf("replay of the reset journal: %v, slot %s; want success and an empty slot", replays[0].err, state.State) + case !tt.resetFirst && (replays[0].err == nil || !bytes.Equal(readSector(t, dev, journalAt), journalLeft)): + t.Errorf("replay of the damaged journal: %v; want an error and the journal untouched", replays[0].err) + } + }) + } +} + +type replay struct { + slot int + err error +} + +// detectFailures runs one failure detection pass on peer and lists the +// journals it replayed. +func detectFailures(peer *node.Registry, dev *storage.Device) []replay { + var replays []replay + peer.SetRecoveryHandler(func(info *node.FailedNodeInfo) error { + _, err := journal.ReplayJournal(context.Background(), &journal.RecoveryContext{Dev: dev}, info) + replays = append(replays, replay{info.SlotIndex, err}) + return err + }) + peer.StartFailureDetection() + peer.Stop() + return replays +} + func newChecker(dev *storage.Device) *fsck.Checker { return fsck.New(dev, fsck.Options{Output: &bytes.Buffer{}}) } diff --git a/internal/node/registry.go b/internal/node/registry.go index 6a0726ca..35fc2975 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -895,6 +895,10 @@ func (r *Registry) clearFailedSlot(slotIdx int, uuid [16]byte) error { return err } +// releaseLeaseBudget is how long ReleaseSlot trusts its recovery lease before +// it frees the slot, well inside recoveryLease so peers' clocks have room. +var releaseLeaseBudget = recoveryLease / 2 + // ReleaseSlot frees slot slotIdx, whose node will not return, if the slot // still holds expected, the raw bytes the caller checked. Like a peer that // replays a crashed node's journal, it first takes the slot's recovery lease, @@ -915,7 +919,8 @@ func (r *Registry) ReleaseSlot(slotIdx int, expected []byte, reset func() error) slot.Magic = SlotMagic slot.State = NodeStateDead slot.RecoveryOwner = r.localUUID - slot.RecoveryStart = time.Now().Unix() + leaseStart := time.Now() + slot.RecoveryStart = leaseStart.Unix() leased := serializeSlot(slot) if ok, err := r.casSlot(slotIdx, expected, leased); err != nil || !ok { return false, err @@ -928,6 +933,10 @@ func (r *Registry) ReleaseSlot(slotIdx int, expected []byte, reset func() error) if err := reset(); err != nil { return false, err } + if age := time.Since(leaseStart); age > releaseLeaseBudget { + return false, fmt.Errorf("the journal reset took %s, too long to trust the slot's recovery lease; "+ + "the slot stays dead until the lease lapses", age.Round(time.Millisecond)) + } ok, err := r.casSlot(slotIdx, leased, serializeSlot(&Slot{Magic: SlotMagic, State: NodeStateEmpty})) if err == nil && !ok { err = fmt.Errorf("slot %d changed while its recovery lease was held", slotIdx) diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index c152e117..b23841c7 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -1195,3 +1195,30 @@ func writeLocalSlotAt(t *testing.T, reg *Registry, idx int, pid int, heartbeat t } return slot } + +func TestReleaseSlotGivesUpOnSlowReset(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + defer func(old time.Duration) { releaseLeaseBudget = old }(releaseLeaseBudget) + releaseLeaseBudget = claimSettle + + reg := NewRegistry(dev, 0) + gone := &Slot{Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{5}, HeartbeatTime: time.Now().Add(-time.Hour).Unix()} + if err := reg.writeSlot(0, gone); err != nil { + t.Fatal(err) + } + raw, _, err := reg.readSlotRaw(0) + if err != nil { + t.Fatal(err) + } + released, err := reg.ReleaseSlot(0, raw, func() error { + time.Sleep(2 * claimSettle) + return nil + }) + if released || err == nil { + t.Fatalf("ReleaseSlot after a slow reset = %v, %v; want an error", released, err) + } + if slot, err := reg.readSlot(0); err != nil || slot.State != NodeStateDead || slot.RecoveryOwner != reg.localUUID { + t.Errorf("slot after the abandoned release: %+v, %v; want dead under this registry's lease", slot, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 9c44c9aa..6f168438 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -889,12 +889,33 @@ scenarios: environments: go: internal/fsck/slots_test.go::TestReleaseSlot - - id: fsck-release-slot-registry-changes - name: fsck slot release stops when the registry changes - description: A node that registers between the release check and the write stops the release before the journal is reset + - id: fsck-release-slot-refusals + name: fsck slot release refusals + description: Release refuses a slot number the registry lacks (ErrNoSuchSlot), a journal a mount can replay unless forced, an unreadable slot, and a superblock mount would reject priority: go_only environments: - go: internal/fsck/slots_test.go::TestReleaseSlotStopsWhenRegistryChanges + go: internal/fsck/slots_test.go::TestReleaseSlotRefusals + + - id: fsck-release-slot-lease + name: fsck slot release holds the recovery lease + description: A node that takes the slot before fsck's lease stops the release with the journal untouched; a node mounting during the lease takes another slot; a slot written during the lease is reported as reset but not free + priority: go_only + environments: + go: internal/fsck/slots_test.go::TestReleaseSlotLease + + - id: fsck-release-slot-crash + name: fsck slot release survives a crash + description: After fsck crashes holding the lease, before or after the journal reset, mounts and peers skip the slot until the lease lapses; then a peer fails on the untouched damaged journal, or frees the slot after replaying the reset one + priority: go_only + environments: + go: internal/fsck/slots_test.go::TestReleaseSlotCrash + + - id: release-slot-slow-reset + name: Slot release gives up on a slow journal reset + description: When the journal reset outlasts half the recovery lease, ReleaseSlot leaves the slot dead under its lease instead of freeing it + priority: go_only + environments: + go: internal/node/registry_test.go::TestReleaseSlotGivesUpOnSlowReset - id: fsck-bad-layout name: fsck stops on a layout it cannot trust