Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions cmd/fsck/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,21 @@
// -repair -dry-run: Report what -repair would change, write nothing
// -force: Keep checking after a fatal error; with -repair, repair even if
// a stopped node's journal still needs replay
// -list-slots: List node registry slots with their lease and journal
// -release-slot N: Free the slot of a node that will not return; with
// -discard-journal, reset its journal and free it
//
// Exit codes, bits as with fsck(8):
//
// 0: No errors (warnings may remain)
// 1: Errors were repaired
// 1: Errors were repaired, or -release-slot -discard-journal freed a slot
// 4: Errors remain on disk
// 8: I/O failure, or the check could not finish or was refused
// 16: Usage error
package main

import (
"errors"
"flag"
"fmt"
"os"
Expand All @@ -38,7 +42,7 @@ func main() {
var (
repair = flag.Bool("repair", false, "Repair problems automatically where possible (see Repair below)")
dryRun = flag.Bool("dry-run", false, "With -repair, report what would change without writing")
force = flag.Bool("force", false, "Keep checking after a fatal error; with -repair, repair even if a stopped node's journal still needs replay")
force = flag.Bool("force", false, "Keep checking after a fatal error; with -repair, repair even if a stopped node's journal still needs replay; with -release-slot, discard a journal a mount could replay")
verbose = flag.Bool("verbose", false, "Enable verbose progress output")
quiet = flag.Bool("quiet", false, "Only print errors and summary")
stopOnFatal = flag.Bool("stop-on-fatal", true, "Stop checking when a fatal error is found")
Expand All @@ -47,6 +51,10 @@ func main() {
summaryOnly = flag.Bool("summary", false, "Only print final summary")
showProblems = flag.Bool("show-problems", true, "Show individual problems found")
showVersion = flag.Bool("version", false, "Print version and exit")

listSlots = flag.Bool("list-slots", false, "List node registry slots with their lease and journal, then exit")
releaseSlotN = flag.Int("release-slot", -1, "Check that registry slot N can be released and print what releasing it discards")
discardJournal = flag.Bool("discard-journal", false, "With -release-slot, reset the slot's journal and free the slot")
)
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <device>\n\n", os.Args[0])
Expand All @@ -58,10 +66,13 @@ func main() {
fmt.Fprintf(os.Stderr, " %s -repair /path/to/disk.img # Check and repair\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s -verbose -repair /dev/sdc # Verbose repair\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s -repair -dry-run /dev/sdc # Show what repair would change\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s -list-slots /dev/sdb # Show node registry slots\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s -release-slot 3 /dev/sdb # Show what releasing slot 3 discards\n", os.Args[0])
fmt.Fprint(os.Stderr, repairHelp)
fmt.Fprint(os.Stderr, slotsHelp)
fmt.Fprintf(os.Stderr, "\nExit codes (bits, as with fsck(8); 5 means some errors were repaired and some remain):\n")
fmt.Fprintf(os.Stderr, " 0: No errors (warnings may remain)\n")
fmt.Fprintf(os.Stderr, " 1: Errors were repaired\n")
fmt.Fprintf(os.Stderr, " 1: Errors were repaired, or -release-slot -discard-journal freed a slot\n")
fmt.Fprintf(os.Stderr, " 4: Errors remain on disk\n")
fmt.Fprintf(os.Stderr, " 8: I/O failure, or the check could not finish or was refused\n")
fmt.Fprintf(os.Stderr, " 16: Usage error\n")
Expand All @@ -88,6 +99,18 @@ func main() {
fmt.Fprintln(os.Stderr, "Error: -dry-run needs -repair")
os.Exit(fsck.ExitUsage)
}
releasing := *releaseSlotN != -1
switch {
case *discardJournal && !releasing:
fmt.Fprintln(os.Stderr, "Error: -discard-journal needs -release-slot")
os.Exit(fsck.ExitUsage)
case releasing && *releaseSlotN < 0:
fmt.Fprintln(os.Stderr, "Error: -release-slot needs a slot number")
os.Exit(fsck.ExitUsage)
case *listSlots && releasing, (*listSlots || releasing) && *repair:
fmt.Fprintln(os.Stderr, "Error: -list-slots, -release-slot and -repair run separately")
os.Exit(fsck.ExitUsage)
}
devicePath := flag.Arg(0)

// Determine mode
Expand All @@ -111,6 +134,26 @@ func main() {
}
defer dev.Close()

if *listSlots || releasing {
checker := fsck.New(dev, fsck.Options{Output: os.Stdout, Verbose: *verbose})
if *listSlots {
err = printSlots(os.Stdout, checker)
} else {
err = releaseSlot(os.Stdout, checker, *releaseSlotN, *discardJournal, *force)
}
switch {
case errors.Is(err, fsck.ErrNoSuchSlot):
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(fsck.ExitUsage)
case err != nil:
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(fsck.ExitOperational)
case *discardJournal:
os.Exit(fsck.ExitCorrected)
}
os.Exit(fsck.ExitClean)
}

// Configure fsck options
opts := fsck.Options{
Mode: mode,
Expand Down
94 changes: 94 additions & 0 deletions cmd/fsck/slots.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package main

import (
"fmt"
"io"
"strings"
"text/tabwriter"
"time"

"github.com/supporttools/GoClusterFS/internal/fsck"
"github.com/supporttools/GoClusterFS/internal/node"
)

const slotsHelp = `
Node registry slots:
Every mounted node holds a registry slot with its own journal. When a node
crashes, a peer replays its journal and frees the slot, or a mount on any
host takes the slot and replays the journal first. A slot whose journal
cannot be replayed, for example because its header is damaged, stays
taken. -list-slots shows each slot that is in use or whose journal needs
attention.

-release-slot N checks that slot N can be freed and prints what freeing it
would discard. It refuses while any node has the filesystem mounted, until
a minute after the slot's lease ended, and, without -force, while a mount
could still replay the journal. Add -discard-journal to reset the slot's
journal and free the slot; fsck then exits 1.
`

// printSlots lists the slots that are in use or whose journal was written
// and is not clean.
func printSlots(out io.Writer, checker *fsck.Checker) error {
slots, err := checker.Slots()
if err != nil {
return err
}
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "SLOT\tSTATE\tHOST\tNODE\tLEASE\tJOURNAL")
shown := 0
for _, s := range slots {
if !s.InUse() && s.JournalFresh {
continue
}
state, host, uuid := "unreadable: "+fmt.Sprint(s.Err), "-", "-"
if s.Slot != nil {
state = s.Slot.State.String()
host = strings.TrimRight(string(s.Slot.Hostname[:]), "\x00")
uuid = fmt.Sprintf("%x", s.Slot.UUID[:8])
}
fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\n", s.Index, state, host, uuid, lease(s), s.Journal)
shown++
}
if err := w.Flush(); err != nil {
return err
}
fmt.Fprintf(out, "\n%d of %d slots shown; the others are free, with a journal that is clean or was never used.\n", shown, len(slots))
return nil
}

// releaseSlot checks slot i and prints what releasing it discards. With
// discard set it also releases the slot.
func releaseSlot(out io.Writer, checker *fsck.Checker, i int, discard, force bool) error {
plan, err := checker.PlanRelease(i, force)
if err != nil {
return err
}
s := plan.Slot
fmt.Fprintf(out, "Slot %d: %s, %s, lease %s\n", s.Index, s.Name(), s.Slot.State, lease(s))
fmt.Fprintf(out, "Journal: %s\n", s.Journal)
fmt.Fprintf(out, "Releasing discards %s.\n", plan.Discards)
if !discard {
fmt.Fprintln(out, "Nothing was written. Run again with -discard-journal to reset the journal and free the slot.")
return nil
}
if err := checker.Release(plan); err != nil {
return err
}
fmt.Fprintf(out, "Released slot %d: its journal is reset and a node can register in it.\n", i)
return nil
}

func lease(s fsck.SlotStatus) string {
switch {
case s.Running:
return "held (node running)"
case s.Slot == nil || s.Slot.State == node.NodeStateEmpty:
return "-"
}
expiry := s.Slot.LeaseExpiry()
if left := time.Until(expiry); left > 0 {
return "ends in " + left.Round(time.Second).String()
}
return "ended " + time.Since(expiry).Round(time.Second).String() + " ago"
}
10 changes: 9 additions & 1 deletion docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,15 @@ A mount exits with `Journal recovery for slot N failed: ...` when it cannot repl

`-skip-recovery` does not stop the node from replaying crashed peers' journals. `-journal=false` also skips the replay of its own slot, and leaves that journal in place for the next mount, so don't use it on a device that may have a dirty journal.

A slot belongs to its host's hostname and boot ID. If the damaged slot belongs to a node that has rebooted or will not mount again, the other nodes log `Replaying slot N ... failed` and retry every minute. Between retries, a new mount on any host can take that slot and then fail with the same error. There is no safe tool to free the slot yet.
A slot belongs to its host's hostname and boot ID. If the damaged slot belongs to a node that has rebooted or will not mount again, the other nodes log `Replaying slot N ... failed` and retry every minute. Between retries, a new mount on any host can take that slot and then fail with the same error. Free the slot with fsck. Unmount the filesystem on every node first, then:

```bash
./goclusterfs-fsck -list-slots /dev/device # find the slot and its journal state
./goclusterfs-fsck -release-slot N /dev/device # check the slot; prints what releasing discards
./goclusterfs-fsck -release-slot N -discard-journal /dev/device # reset its journal and free the slot
```

`-release-slot` refuses while any node has the filesystem mounted, and until a minute after the slot's lease ended: its last heartbeat plus 10 seconds, or for a slot another node was replaying, that node's last lease renewal plus 60 seconds. Without `-force` it also refuses a journal that a mount might still replay: one with committed transactions that fsck reads in full, or one fsck cannot read to the end. Mount a node to replay it instead, and add `-force` only if that replay fails. With `-discard-journal`, fsck holds the slot's recovery lease while it resets the journal, frees the slot, and exits 1. The next node to mount can then use the slot. Run fsck afterwards, because discarded transactions may have been only partly applied.

### Emergency: All Nodes Down

Expand Down
8 changes: 8 additions & 0 deletions internal/fsck/export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,11 @@ func SetBeforeRepairWrites(f func()) (restore func()) {
beforeRepairWrites = f
return func() { beforeRepairWrites = old }
}

// SetReleaseHooks runs before once Release has checked the registry, and
// during once Release holds the slot's recovery lease.
func SetReleaseHooks(before, during func()) (restore func()) {
oldBefore, oldDuring := beforeReleaseWrites, whileReleaseHoldsLease
beforeReleaseWrites, whileReleaseHoldsLease = before, during
return func() { beforeReleaseWrites, whileReleaseHoldsLease = oldBefore, oldDuring }
}
2 changes: 1 addition & 1 deletion internal/fsck/repair.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ func (c *Checker) journalNeedsReplay(i int) string {
if sb.MaxNodes == 0 || uint32(i) >= sb.MaxNodes {
return "has no journal slot"
}
off := sb.JournalRegionOffset + uint64(i)*(sb.JournalRegionSize/uint64(sb.MaxNodes))
off, _ := c.journalArea(i)
buf := storage.NewAlignedBuffer(storage.SectorSize).Bytes()
if err := c.readSector(buf, off); err != nil {
return "has an unreadable journal: " + err.Error()
Expand Down
Loading
Loading