From badfb5cb1ca982d9e5d54abb36d5329f682c0c8e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 05:02:49 -0600 Subject: [PATCH 1/2] fix(journal): bound the recovery scan by the bytes between head and tail Both recovery scans ran until the position equaled the tail. A tail no record boundary reaches, such as a corrupt in-memory pointer, sent them around the ring through stale records forever, replaying old transactions on the way. They now stop after reading the bytes from head to tail, so every scan ends and reads each record at most once. Co-Authored-By: Claude Opus 5.5 --- internal/journal/checkpoint_test.go | 35 +++++++++++++++ internal/journal/recovery.go | 68 +++++++++-------------------- 2 files changed, 56 insertions(+), 47 deletions(-) diff --git a/internal/journal/checkpoint_test.go b/internal/journal/checkpoint_test.go index 948c42fe..2588877c 100644 --- a/internal/journal/checkpoint_test.go +++ b/internal/journal/checkpoint_test.go @@ -158,3 +158,38 @@ func TestRecoveryEndsWithTailAtEnd(t *testing.T) { t.Fatal("recovery scan still running after 10s with the tail at the end of the data area") } } + +// A tail that no record boundary reaches, from a corrupt pointer, must not keep the scan reading the ring forever. +func TestScanStopsAtUnreachableTail(t *testing.T) { + j := newTestJournal(t) + for j.region.Superblock().WrapCount == 0 { + commitIntents(t, j, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + } + pending := commitIntents(t, j, false, false) + region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize) + if err != nil { + t.Fatal(err) + } + region.superblock.TailOffset += 512 + + done := make(chan []uint64, 1) + go func() { + var replayed []uint64 + _ = scanJournal(region, func(txn *RecoveredTxn) error { + replayed = append(replayed, txn.Header.Seq) + return nil + }, &RecoveryStats{}, nil, nil) + done <- replayed + }() + select { + case got := <-done: + if !slices.Equal(got, pending) { + t.Errorf("replayed %v, want each pending intent once: %v", got, pending) + } + case <-time.After(10 * time.Second): + t.Fatal("scan still running after 10s with an unreachable tail") + } +} diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index d4a0bc8e..53da5b43 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -106,20 +106,11 @@ func RecoverChecked(region *Region, handler RecoveryHandler, check func() error) // This is Phase 1 of the two-phase recovery protocol. // Records are identified by IntentCompletionMagic (0x43544E49 "INTC"). func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, completed map[uint64]bool, stats *RecoveryStats) error { + span := ringSpan(head, tail, dataAreaSize) pos := head - for pos != tail { - // Handle wrap-around - if pos >= dataAreaSize { - pos = 0 - } - - // Calculate bytes remaining to scan - var remaining uint32 - if tail > pos { - remaining = tail - pos - } else { - remaining = dataAreaSize - pos + tail - } + for scanned := uint32(0); scanned < span; { + pos %= dataAreaSize + remaining := span - scanned if remaining < 4 { // Not enough data for even a magic number @@ -144,7 +135,7 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet stats.IntentCompletions++ } // Advance by one aligned sector (IntentCompletion is padded to 512 bytes) - pos += storage.SectorSize + pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize } else if magic == TxnMagic { // This is a transaction - skip it (we'll process in phase 2) header, err := DeserializeTxnHeader(sectorBuf[:TxnHeaderSize]) @@ -153,21 +144,24 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet } // Advance by aligned transaction length alignedLen := ((header.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos += alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen } else { // Unknown record type - corruption or end of valid data break } - - // Handle wrap-around - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } } return nil } +// ringSpan returns the bytes from head to tail; a tail at the end of the data area is the end, not 0. +func ringSpan(head, tail, dataAreaSize uint32) uint32 { + if tail >= head { + return tail - head + } + return dataAreaSize - head + tail +} + // scanJournal scans the journal from head to tail, recovering transactions. // It implements the two-phase intent logging recovery protocol: // Phase 1: Scan for IntentCompletion records to find completed intents @@ -194,20 +188,11 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, } // Phase 2: Replay transactions that are not completed + span := ringSpan(head, tail, dataAreaSize) pos := head - for pos != tail { - // Handle wrap-around - if pos >= dataAreaSize { - pos = 0 - } - - // Calculate bytes remaining to scan - var remaining uint32 - if tail > pos { - remaining = tail - pos - } else { - remaining = dataAreaSize - pos + tail - } + for scanned := uint32(0); scanned < span; { + pos %= dataAreaSize + remaining := span - scanned if remaining < TxnHeaderSize { // Not enough data for a transaction header @@ -237,10 +222,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Skip IntentCompletion records (already processed in Phase 1) if magic == IntentCompletionMagic { - pos += storage.SectorSize - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } + pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize continue } @@ -295,11 +277,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, stats.IntentsSkipped++ // Skip to next record without replaying alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos = pos + alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen stats.BytesScanned += alignedLen - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } continue } @@ -344,13 +323,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Advance position (account for sector alignment) alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos = pos + alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen stats.BytesScanned += alignedLen - - // Handle wrap-around - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } } return nil From f6535aa53eb47ff4c92fc925e4b85667c21bd42e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 05:08:02 -0600 Subject: [PATCH 2/2] test: track the bounded recovery scan in the manifest Co-Authored-By: Claude Opus 5.5 --- tests/test-manifest.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 66b65ad6..d1be9667 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -728,6 +728,13 @@ scenarios: environments: go: internal/journal/checkpoint_test.go::TestRecoveryEndsWithTailAtEnd + - id: recovery-scan-bounded + name: Recovery scans stop after the bytes between head and tail + description: With a tail no record boundary reaches and stale records around the ring, both recovery scans end and replay each pending transaction once + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestScanStopsAtUnreachableTail + - id: create-replays-after-checkpoint-crash name: A create cut off after a checkpoint is replayed description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean