diff --git a/internal/journal/checkpoint_test.go b/internal/journal/checkpoint_test.go index 948c42fe..2588877c 100644 --- a/internal/journal/checkpoint_test.go +++ b/internal/journal/checkpoint_test.go @@ -158,3 +158,38 @@ func TestRecoveryEndsWithTailAtEnd(t *testing.T) { t.Fatal("recovery scan still running after 10s with the tail at the end of the data area") } } + +// A tail that no record boundary reaches, from a corrupt pointer, must not keep the scan reading the ring forever. +func TestScanStopsAtUnreachableTail(t *testing.T) { + j := newTestJournal(t) + for j.region.Superblock().WrapCount == 0 { + commitIntents(t, j, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + } + pending := commitIntents(t, j, false, false) + region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize) + if err != nil { + t.Fatal(err) + } + region.superblock.TailOffset += 512 + + done := make(chan []uint64, 1) + go func() { + var replayed []uint64 + _ = scanJournal(region, func(txn *RecoveredTxn) error { + replayed = append(replayed, txn.Header.Seq) + return nil + }, &RecoveryStats{}, nil, nil) + done <- replayed + }() + select { + case got := <-done: + if !slices.Equal(got, pending) { + t.Errorf("replayed %v, want each pending intent once: %v", got, pending) + } + case <-time.After(10 * time.Second): + t.Fatal("scan still running after 10s with an unreachable tail") + } +} diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index d4a0bc8e..53da5b43 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -106,20 +106,11 @@ func RecoverChecked(region *Region, handler RecoveryHandler, check func() error) // This is Phase 1 of the two-phase recovery protocol. // Records are identified by IntentCompletionMagic (0x43544E49 "INTC"). func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, completed map[uint64]bool, stats *RecoveryStats) error { + span := ringSpan(head, tail, dataAreaSize) pos := head - for pos != tail { - // Handle wrap-around - if pos >= dataAreaSize { - pos = 0 - } - - // Calculate bytes remaining to scan - var remaining uint32 - if tail > pos { - remaining = tail - pos - } else { - remaining = dataAreaSize - pos + tail - } + for scanned := uint32(0); scanned < span; { + pos %= dataAreaSize + remaining := span - scanned if remaining < 4 { // Not enough data for even a magic number @@ -144,7 +135,7 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet stats.IntentCompletions++ } // Advance by one aligned sector (IntentCompletion is padded to 512 bytes) - pos += storage.SectorSize + pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize } else if magic == TxnMagic { // This is a transaction - skip it (we'll process in phase 2) header, err := DeserializeTxnHeader(sectorBuf[:TxnHeaderSize]) @@ -153,21 +144,24 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet } // Advance by aligned transaction length alignedLen := ((header.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos += alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen } else { // Unknown record type - corruption or end of valid data break } - - // Handle wrap-around - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } } return nil } +// ringSpan returns the bytes from head to tail; a tail at the end of the data area is the end, not 0. +func ringSpan(head, tail, dataAreaSize uint32) uint32 { + if tail >= head { + return tail - head + } + return dataAreaSize - head + tail +} + // scanJournal scans the journal from head to tail, recovering transactions. // It implements the two-phase intent logging recovery protocol: // Phase 1: Scan for IntentCompletion records to find completed intents @@ -194,20 +188,11 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, } // Phase 2: Replay transactions that are not completed + span := ringSpan(head, tail, dataAreaSize) pos := head - for pos != tail { - // Handle wrap-around - if pos >= dataAreaSize { - pos = 0 - } - - // Calculate bytes remaining to scan - var remaining uint32 - if tail > pos { - remaining = tail - pos - } else { - remaining = dataAreaSize - pos + tail - } + for scanned := uint32(0); scanned < span; { + pos %= dataAreaSize + remaining := span - scanned if remaining < TxnHeaderSize { // Not enough data for a transaction header @@ -237,10 +222,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Skip IntentCompletion records (already processed in Phase 1) if magic == IntentCompletionMagic { - pos += storage.SectorSize - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } + pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize continue } @@ -295,11 +277,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, stats.IntentsSkipped++ // Skip to next record without replaying alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos = pos + alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen stats.BytesScanned += alignedLen - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } continue } @@ -344,13 +323,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Advance position (account for sector alignment) alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize - pos = pos + alignedLen + pos, scanned = pos+alignedLen, scanned+alignedLen stats.BytesScanned += alignedLen - - // Handle wrap-around - if pos >= dataAreaSize && pos != tail { - pos = pos - dataAreaSize - } } return nil diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 66b65ad6..d1be9667 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -728,6 +728,13 @@ scenarios: environments: go: internal/journal/checkpoint_test.go::TestRecoveryEndsWithTailAtEnd + - id: recovery-scan-bounded + name: Recovery scans stop after the bytes between head and tail + description: With a tail no record boundary reaches and stale records around the ring, both recovery scans end and replay each pending transaction once + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestScanStopsAtUnreachableTail + - id: create-replays-after-checkpoint-crash name: A create cut off after a checkpoint is replayed description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean