From a26e4424af7897b2b362fdb78ed0f0a30d82429d Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:42:49 -0500 Subject: [PATCH] fix(mount): make -skip-recovery name the slot whose journal it discards -skip-recovery discarded the journal of whichever slot the mount took. While a peer held this host's damaged slot, the mount could take another crashed node's slot and overwrite that node's unreplayed journal. Checking that the slot was this host's did not help: a failed mount stamps the slot with this host's name first, so a retry passed. -skip-recovery=N now discards a journal only when the mount takes slot N, the slot named in the recovery error. A mount that takes any other slot logs that it ignores the flag and replays that slot's journal. A bare -skip-recovery is a usage error. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 2 +- cmd/mount/main.go | 16 ++-- docs/API-CONFIG.md | 2 +- docs/OPERATIONS.md | 13 ++- docs/TROUBLESHOOTING.md | 4 +- tests/integration/journal_slots_test.go | 111 +++++++++++++++++++++++- tests/test-manifest.yaml | 7 ++ 8 files changed, 139 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3fd4906..f53777a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Run `goclusterfs-fsck -repair` before the first mount after upgrading an image that has snapshots or clones. Unmount the device on every node first. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. Until `goclusterfs-fsck -repair` counts them, this version never frees such blocks, so deleting those files leaks their space, and it refuses to snapshot or clone them (`ENOTSUP`). In-place writes to an old clone still change its source. - Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. +- `-skip-recovery` now takes the slot number from the recovery error, as `-skip-recovery=N`, and discards a journal only when the mount takes slot N. A bare `-skip-recovery` is a usage error. ### Added diff --git a/README.md b/README.md index 27a401fe..9eb0782a 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ These are the `goclusterfs-mount` flags. Run `./goclusterfs-mount -h` for the fu | Flag | Default | Description | |------|---------|-------------| | `-journal` | `true` | Enable write-ahead journal | -| `-skip-recovery` | `false` | Skip journal recovery on mount (dangerous) | +| `-skip-recovery=N` | `-1` (off) | Discard the journal of registry slot N instead of replaying it, if the mount takes slot N (dangerous) | ### Network Broadcast Options diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 35c8d5a8..ed28fc91 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -45,7 +45,7 @@ func main() { // Journal configuration journalEnabled = flag.Bool("journal", true, "Enable write-ahead journal for crash consistency") - skipRecovery = flag.Bool("skip-recovery", false, "Skip journal recovery (emergency maintenance only)") + skipRecovery = flag.Int("skip-recovery", -1, "Discard the journal of registry slot N instead of replaying it, if this mount takes slot N (emergency maintenance only)") // Network broadcast configuration broadcastEnabled = flag.Bool("broadcast", true, "Enable network cache invalidation broadcast") @@ -152,7 +152,13 @@ func main() { // Check for and recover any dirty journal from a previous crash // This must happen BEFORE we create our own journal to avoid data loss - if *journalEnabled && !*skipRecovery { + // -skip-recovery names the slot whose journal the operator chose to discard; + // a mount that takes any other slot replays that slot's journal. + skip := *skipRecovery >= 0 && *skipRecovery == slotIdx + if *skipRecovery >= 0 && !skip { + log.Printf("Journal: ignoring -skip-recovery=%d: this mount took slot %d, so it replays that slot's journal", *skipRecovery, slotIdx) + } + if *journalEnabled && !skip { // Set up recovery context with our managers recoveryCtx := &journal.RecoveryContext{ Dev: dev, @@ -167,14 +173,14 @@ func main() { // Replay a dirty journal left in this slot's region by its previous owner result, err := journal.ScanAndRecover(dev, journalOffset, journalSize, recoveryHandler) if err != nil { - log.Fatalf("Journal recovery for slot %d failed: %v (mount with -skip-recovery to discard this journal)", slotIdx, err) + log.Fatalf("Journal recovery for slot %d failed: %v (to discard this journal, mount with -skip-recovery=%d)", slotIdx, err, slotIdx) } if result != nil && result.WasNeeded { log.Printf("Journal recovery: replayed %d committed transactions (%d bytes scanned) from crashed node %x", result.Stats.TxnCommitted, result.Stats.BytesScanned, result.NodeUUID[:8]) } - } else if *skipRecovery { - log.Println("Journal: WARNING - skipping recovery (emergency mode)") + } else if skip { + log.Printf("Journal: WARNING - discarding the journal of slot %d (-skip-recovery)", slotIdx) } releasePrevious() diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 26190947..d1558c95 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -53,7 +53,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For | Flag | Type | Default | Description | |------|------|---------|-------------| | `-journal` | bool | `true` | Enable write-ahead journal for crash consistency | -| `-skip-recovery` | bool | `false` | Skip journal recovery on mount (dangerous, may lose data) | +| `-skip-recovery` | int | `-1` (off) | Discard the journal of registry slot N instead of replaying it, if the mount takes slot N (dangerous, may lose data) | ### Network Broadcast Flags diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 6e7a8008..56ff789f 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -277,16 +277,15 @@ goclusterfs-mount -device /dev/device -mount /mnt/cluster A mount exits with `Journal recovery for slot N failed: ...` when it cannot replay the journal in the slot it took. Read the rest of the error first: - **Storage or replay errors** (an I/O error, or `journal recovery failed: ...`): the journal stays in recovery. Fix the storage and mount again without `-skip-recovery`; the replay resumes. -- **A damaged journal header** (`journal superblock checksum mismatch`, `invalid journal magic`, or `journal is corrupt`): the journal cannot be replayed. Only then use `-skip-recovery`, which discards the journal. +- **A damaged journal header** (`journal superblock checksum mismatch`, `invalid journal magic`, or `journal is corrupt`): the journal cannot be replayed. Only then use `-skip-recovery=N`, with the slot number N from the error, which discards slot N's journal. -`-skip-recovery` discards the journal of whichever slot the mount takes, and that is not always the damaged one. About 20 seconds after the failed mount, a running node tries to replay the damaged slot and holds it for 60 seconds. A mount in that window takes a different slot, which may hold another crashed node's journal that has not been replayed yet. To discard the right journal: +`-skip-recovery=N` discards a journal only when the mount takes slot N. If it takes another slot, it logs `ignoring -skip-recovery=N` and replays that slot's journal as usual, so it never discards a journal you did not name. A mount takes another slot while a running node is replaying slot N: about 20 seconds after the failed mount, a running node tries to replay the damaged slot, holds it for 60 seconds, and tries again every minute. To discard slot N's journal: -1. Either run step 2 right after the failed mount, within about 20 seconds, or first unmount on every other node and wait one minute. -2. On the same host, before it reboots, mount once with `-skip-recovery`. -3. Check the mount's `Registered as node N` log line. N must be the slot from the error. If it is not, unmount at once: the mount discarded another slot's journal. Check that node's recent changes, and restore them from backup if needed. -4. Unmount on every node and run fsck, because discarded transactions may have been only partly applied. +1. Either mount right after the failed mount, within about 20 seconds, or first unmount on every other node and wait one minute. +2. On the same host, before it reboots, mount once with `-skip-recovery=N`. The mount logs `discarding the journal of slot N`. +3. Unmount on every node and run fsck, because discarded transactions may have been only partly applied. -`-skip-recovery` does not stop the node from replaying crashed peers' journals. `-journal=false` also skips the replay of its own slot, and leaves that journal in place for the next mount, so don't use it on a device that may have a dirty journal. +`-skip-recovery=N` does not stop the node from replaying crashed peers' journals. `-journal=false` also skips the replay of its own slot, and leaves that journal in place for the next mount, so don't use it on a device that may have a dirty journal. A slot belongs to its host's hostname and boot ID. If the damaged slot belongs to a node that has rebooted or will not mount again, the other nodes log `Replaying slot N ... failed` and retry every minute. Between retries, a new mount on any host can take that slot and then fail with the same error. There is no safe tool to free the slot yet. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 9735a04b..2ab44126 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -147,14 +147,14 @@ A crashed node's locks free up on their own: other nodes take them over once the ### "Journal recovery for slot N failed" -**Symptom**: Mount exits with `Journal recovery for slot N failed: ... (mount with -skip-recovery to discard this journal)`. +**Symptom**: Mount exits with `Journal recovery for slot N failed: ... (to discard this journal, mount with -skip-recovery=N)`. **Cause**: The mount could not replay the journal in the registry slot it took. A normal crash does not cause this: the next mount on the same host replays the journal by itself. Read the rest of the error: - An I/O error, or `journal recovery failed: ...`: the storage failed or a transaction could not be applied. The journal stays in recovery. Fix the storage and mount again without `-skip-recovery`; the replay resumes. - `journal superblock checksum mismatch`, `invalid journal magic`, or `journal is corrupt`: the journal header is damaged and cannot be replayed. -**Solution**: Only for a damaged header, discard the journal with `-skip-recovery`. That flag discards the journal of whichever slot the mount takes, which may belong to another crashed node, so follow the steps in [Damaged Journal Slot](OPERATIONS.md#damaged-journal-slot) exactly. +**Solution**: Only for a damaged header, discard the journal with `-skip-recovery=N`. The mount discards it only if it takes slot N again; follow the steps in [Damaged Journal Slot](OPERATIONS.md#damaged-journal-slot). ### Another Node's Change Shows Up Late diff --git a/tests/integration/journal_slots_test.go b/tests/integration/journal_slots_test.go index 41928dc5..bcaa9635 100644 --- a/tests/integration/journal_slots_test.go +++ b/tests/integration/journal_slots_test.go @@ -17,7 +17,9 @@ import ( "github.com/supporttools/GoClusterFS/internal/inode" "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" ) var ( @@ -193,7 +195,7 @@ func TestMountRefusesDamagedJournal(t *testing.T) { skipped := &syncBuffer{} node.Log = skipped - node.ExtraArgs = []string{"-skip-recovery"} + node.ExtraArgs = []string{"-skip-recovery=" + slot} if err := cluster.StartNode(0, false); err != nil { t.Fatalf("mount with -skip-recovery: %v", err) } @@ -211,6 +213,113 @@ func TestMountRefusesDamagedJournal(t *testing.T) { } } +// TestSkipRecoveryDiscardsOnlyNamedSlot damages the journal of another host's +// crashed node, whose slot the mount then takes. -skip-recovery naming a +// different slot must not discard it, also on a retry, when the slot already +// carries this host's name from the failed mount. Naming the slot does. +func TestSkipRecoveryDiscardsOnlyNamedSlot(t *testing.T) { + cluster := NewMultiNodeCluster(t, 1, 50) + mfs := cluster.Node(0) + formatted := &syncBuffer{} + mfs.Log = formatted + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("format and start: %v", err) + } + slot := mustMatch(t, slotRe, formatted, "registration") + offset := mustMatch(t, journalInitRe, formatted, "journal init") + if err := cluster.KillNode(0); err != nil { + t.Fatalf("kill: %v", err) + } + moveSlotToOtherHost(t, cluster.ImagePath, slot) + damageJournalHeader(t, cluster.ImagePath, offset) + damaged := readImageSector(t, cluster.ImagePath, offset) + + other := "1" + if slot == other { + other = "2" + } + for attempt := 1; attempt <= 2; attempt++ { + out := &syncBuffer{} + mfs.Log = out + mfs.ExtraArgs = []string{"-skip-recovery=" + other} + if err := cluster.StartNode(0, false); err == nil { + t.Fatalf("attempt %d: -skip-recovery=%s mounted over slot %s's damaged journal", attempt, other, slot) + } + for _, want := range []string{ + "ignoring -skip-recovery=" + other + ": this mount took slot " + slot, + "Journal recovery for slot " + slot + " failed", + "mount with -skip-recovery=" + slot, + } { + if !strings.Contains(out.String(), want) { + t.Errorf("attempt %d: mount output lacks %q:\n%s", attempt, want, out.String()) + } + } + if !bytes.Equal(readImageSector(t, cluster.ImagePath, offset), damaged) { + t.Fatalf("attempt %d: slot %s's journal changed", attempt, slot) + } + } + + named := &syncBuffer{} + mfs.Log = named + mfs.ExtraArgs = []string{"-skip-recovery=" + slot} + if err := cluster.StartNode(0, false); err != nil { + t.Fatalf("mount with -skip-recovery=%s: %v", slot, err) + } + if got := mustMatch(t, journalInitRe, named, "journal init"); got != offset { + t.Errorf("-skip-recovery=%s started a journal at offset %s, want %s", slot, got, offset) + } +} + +// moveSlotToOtherHost makes slot look like the slot of a node on another host +// that crashed an hour ago. +func moveSlotToOtherHost(t *testing.T, imagePath, slot string) { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatalf("open image: %v", err) + } + defer func() { _ = dev.Close() }() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatalf("read superblock: %v", err) + } + i, err := strconv.Atoi(slot) + if err != nil { + t.Fatalf("parse slot %q: %v", slot, err) + } + + off := int64(sb.NodeRegistryOffset) + int64(i)*node.SlotSize + buf := storage.NewAlignedBuffer(node.SlotSize).Bytes() + if _, err := dev.ReadAt(buf, off); err != nil { + t.Fatalf("read slot %d: %v", i, err) + } + binary.LittleEndian.PutUint64(buf[32:40], uint64(time.Now().Add(-time.Hour).Unix())) + clear(buf[64:128]) + copy(buf[64:128], "other-host") + binary.LittleEndian.PutUint32(buf[node.SlotSize-4:], storage.CRC32C(buf[:node.SlotSize-4])) + if _, err := dev.WriteAt(buf, off); err != nil { + t.Fatalf("write slot %d: %v", i, err) + } +} + +func readImageSector(t *testing.T, imagePath, offset string) []byte { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatalf("open image: %v", err) + } + defer func() { _ = dev.Close() }() + off, err := strconv.ParseInt(offset, 10, 64) + if err != nil { + t.Fatalf("parse offset %q: %v", offset, err) + } + buf := storage.NewAlignedBuffer(storage.SectorSize).Bytes() + if _, err := dev.ReadAt(buf, off); err != nil { + t.Fatalf("read offset %d: %v", off, err) + } + return buf +} + func damageJournalHeader(t *testing.T, imagePath, offset string) { t.Helper() dev, err := storage.Open(imagePath) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 8bb1a635..569bcd2b 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -489,6 +489,13 @@ scenarios: environments: go: journal_slots_test.go::TestMountRefusesDamagedJournal + - id: skip-recovery-discards-only-named-slot + name: -skip-recovery discards only the named slot + description: A mount that takes another host's crashed slot ignores -skip-recovery naming a different slot and replays that slot, also on a retry after its own failed mount stamped the slot; -skip-recovery naming the slot discards its journal + priority: go_only + environments: + go: journal_slots_test.go::TestSkipRecoveryDiscardsOnlyNamedSlot + - id: journal-scan-never-initialized name: Recovery scan of a never-initialized journal description: ScanAndRecover treats an all-zero journal header as nothing to recover