Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions internal/journal/checkpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,3 +158,38 @@ func TestRecoveryEndsWithTailAtEnd(t *testing.T) {
t.Fatal("recovery scan still running after 10s with the tail at the end of the data area")
}
}

// A tail that no record boundary reaches, from a corrupt pointer, must not keep the scan reading the ring forever.
func TestScanStopsAtUnreachableTail(t *testing.T) {
j := newTestJournal(t)
for j.region.Superblock().WrapCount == 0 {
commitIntents(t, j, true)
if _, err := j.Checkpoint(); err != nil {
t.Fatal(err)
}
}
pending := commitIntents(t, j, false, false)
region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize)
if err != nil {
t.Fatal(err)
}
region.superblock.TailOffset += 512

done := make(chan []uint64, 1)
go func() {
var replayed []uint64
_ = scanJournal(region, func(txn *RecoveredTxn) error {
replayed = append(replayed, txn.Header.Seq)
return nil
}, &RecoveryStats{}, nil, nil)
done <- replayed
}()
select {
case got := <-done:
if !slices.Equal(got, pending) {
t.Errorf("replayed %v, want each pending intent once: %v", got, pending)
}
case <-time.After(10 * time.Second):
t.Fatal("scan still running after 10s with an unreachable tail")
}
}
68 changes: 21 additions & 47 deletions internal/journal/recovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,20 +106,11 @@ func RecoverChecked(region *Region, handler RecoveryHandler, check func() error)
// This is Phase 1 of the two-phase recovery protocol.
// Records are identified by IntentCompletionMagic (0x43544E49 "INTC").
func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, completed map[uint64]bool, stats *RecoveryStats) error {
span := ringSpan(head, tail, dataAreaSize)
pos := head
for pos != tail {
// Handle wrap-around
if pos >= dataAreaSize {
pos = 0
}

// Calculate bytes remaining to scan
var remaining uint32
if tail > pos {
remaining = tail - pos
} else {
remaining = dataAreaSize - pos + tail
}
for scanned := uint32(0); scanned < span; {
pos %= dataAreaSize
remaining := span - scanned

if remaining < 4 {
// Not enough data for even a magic number
Expand All @@ -144,7 +135,7 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet
stats.IntentCompletions++
}
// Advance by one aligned sector (IntentCompletion is padded to 512 bytes)
pos += storage.SectorSize
pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize
} else if magic == TxnMagic {
// This is a transaction - skip it (we'll process in phase 2)
header, err := DeserializeTxnHeader(sectorBuf[:TxnHeaderSize])
Expand All @@ -153,21 +144,24 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet
}
// Advance by aligned transaction length
alignedLen := ((header.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize
pos += alignedLen
pos, scanned = pos+alignedLen, scanned+alignedLen
} else {
// Unknown record type - corruption or end of valid data
break
}

// Handle wrap-around
if pos >= dataAreaSize && pos != tail {
pos = pos - dataAreaSize
}
}

return nil
}

// ringSpan returns the bytes from head to tail; a tail at the end of the data area is the end, not 0.
func ringSpan(head, tail, dataAreaSize uint32) uint32 {
if tail >= head {
return tail - head
}
return dataAreaSize - head + tail
}

// scanJournal scans the journal from head to tail, recovering transactions.
// It implements the two-phase intent logging recovery protocol:
// Phase 1: Scan for IntentCompletion records to find completed intents
Expand All @@ -194,20 +188,11 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats,
}

// Phase 2: Replay transactions that are not completed
span := ringSpan(head, tail, dataAreaSize)
pos := head
for pos != tail {
// Handle wrap-around
if pos >= dataAreaSize {
pos = 0
}

// Calculate bytes remaining to scan
var remaining uint32
if tail > pos {
remaining = tail - pos
} else {
remaining = dataAreaSize - pos + tail
}
for scanned := uint32(0); scanned < span; {
pos %= dataAreaSize
remaining := span - scanned

if remaining < TxnHeaderSize {
// Not enough data for a transaction header
Expand Down Expand Up @@ -237,10 +222,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats,

// Skip IntentCompletion records (already processed in Phase 1)
if magic == IntentCompletionMagic {
pos += storage.SectorSize
if pos >= dataAreaSize && pos != tail {
pos = pos - dataAreaSize
}
pos, scanned = pos+storage.SectorSize, scanned+storage.SectorSize
continue
}

Expand Down Expand Up @@ -295,11 +277,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats,
stats.IntentsSkipped++
// Skip to next record without replaying
alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize
pos = pos + alignedLen
pos, scanned = pos+alignedLen, scanned+alignedLen
stats.BytesScanned += alignedLen
if pos >= dataAreaSize && pos != tail {
pos = pos - dataAreaSize
}
continue
}

Expand Down Expand Up @@ -344,13 +323,8 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats,

// Advance position (account for sector alignment)
alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize
pos = pos + alignedLen
pos, scanned = pos+alignedLen, scanned+alignedLen
stats.BytesScanned += alignedLen

// Handle wrap-around
if pos >= dataAreaSize && pos != tail {
pos = pos - dataAreaSize
}
}

return nil
Expand Down
7 changes: 7 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,13 @@ scenarios:
environments:
go: internal/journal/checkpoint_test.go::TestRecoveryEndsWithTailAtEnd

- id: recovery-scan-bounded
name: Recovery scans stop after the bytes between head and tail
description: With a tail no record boundary reaches and stale records around the ring, both recovery scans end and replay each pending transaction once
priority: go_only
environments:
go: internal/journal/checkpoint_test.go::TestScanStopsAtUnreachableTail

- id: create-replays-after-checkpoint-crash
name: A create cut off after a checkpoint is replayed
description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean
Expand Down
Loading