From 97b1df1a94ace5ad2dac35bfc0d7a4ef55b27a10 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 00:34:34 -0600 Subject: [PATCH 01/10] test(journal): replay intents a checkpoint passed before they completed Commit takes a transaction out of activeTxns before the operation runs, so a checkpoint can move the head past an intent that is committed but not yet applied, and a crash then loses the operation. These tests stop between Commit and CompleteIntent, checkpoint, and check the next mount replays the intent: in the journal alone, through a create on the filesystem followed by fsck, and across an unmount. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 6 ++ internal/goclusterfs/journal_crash_test.go | 59 ++++++++++++++ internal/journal/checkpoint_test.go | 94 ++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 internal/goclusterfs/journal_crash_test.go create mode 100644 internal/journal/checkpoint_test.go diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a1db6fa4..a55cb6e9 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -694,6 +694,9 @@ func (f *FS) Create(ctx context.Context, name string, flags uint32, mode uint32, return nil, nil, 0, syscall.EAGAIN } +// afterCreateCommit runs between a root create's commit and its first change; tests stop there as a crash would. +var afterCreateCommit func() + // createFileInternal implements file creation for the root directory. func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (*fs.Inode, fs.FileHandle, uint32, syscall.Errno) { inodeNum, err := f.inodeTbl.FindFreeSlot() @@ -745,6 +748,9 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, if commitErr != nil { return nil, nil, 0, syscall.EIO } + if afterCreateCommit != nil { + afterCreateCommit() + } } // === End WAL transaction - intent is now durable === diff --git a/internal/goclusterfs/journal_crash_test.go b/internal/goclusterfs/journal_crash_test.go new file mode 100644 index 00000000..f25cd0b7 --- /dev/null +++ b/internal/goclusterfs/journal_crash_test.go @@ -0,0 +1,59 @@ +package goclusterfs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +// A checkpoint between a create's commit and its changes, then a crash: the next mount must still replay the create. +func TestCreateReplaysAfterCheckpointAndCrash(t *testing.T) { + path := newFormattedImage(t) + fsys := openFormattedFS(t, path) + ctx := context.Background() + if err := fsys.InitRoot(ctx); err != nil { + t.Fatal(err) + } + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + offset, size := sb.JournalRegionOffset, sb.JournalSlotSize() + j, err := journal.Init(fsys.dev, offset, size, fsys.nodeUUID, 1, journal.DefaultConfig()) + if err != nil { + t.Fatal(err) + } + fsys.journal = j + + crashed := make(chan struct{}) + afterCreateCommit = func() { + if _, err := j.Checkpoint(); err != nil { + t.Errorf("checkpoint: %v", err) + } + close(crashed) + select {} + } + t.Cleanup(func() { afterCreateCommit = nil }) + go fsys.Create(ctx, "crashed.txt", 0, 0o644, &fuse.EntryOut{}) + <-crashed + + remounted := openFormattedFS(t, path) + replay := journal.CreateTxnRecoveryHandler(&journal.RecoveryContext{ + Dev: remounted.dev, InodeTbl: remounted.inodeTbl, BitmapMgr: remounted.bitmapMgr, + DirMgr: remounted.dirMgr, ExtentMgr: remounted.extentMgr, + }) + result, err := journal.ScanAndRecover(remounted.dev, offset, size, replay) + if err != nil { + t.Fatalf("replay: %v", err) + } + t.Logf("replay: %+v", result) + lookupFile(t, remounted, "crashed.txt") + if r := runFsck(t, remounted, fsck.DefaultOptions()); len(r.Problems) > 0 { + t.Fatalf("fsck after replay:%s", problemList(r)) + } +} diff --git a/internal/journal/checkpoint_test.go b/internal/journal/checkpoint_test.go new file mode 100644 index 00000000..dd059be1 --- /dev/null +++ b/internal/journal/checkpoint_test.go @@ -0,0 +1,94 @@ +package journal + +import ( + "slices" + "testing" +) + +// commitIntents commits one transaction per entry and completes those marked true. +func commitIntents(t *testing.T, j *Journal, complete ...bool) []uint64 { + t.Helper() + var seqs []uint64 + for i, done := range complete { + txn := j.Begin() + if err := txn.LogInodeCreate(testInodePayload(uint64(100 + i))); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if err != nil { + t.Fatal(err) + } + if done { + if err := j.CompleteIntent(seq); err != nil { + t.Fatal(err) + } + } + seqs = append(seqs, seq) + } + return seqs +} + +// replayedAfterCrash reopens the region as the next mount would and returns the transactions it replays. +func replayedAfterCrash(t *testing.T, j *Journal) []uint64 { + t.Helper() + region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize) + if err != nil { + t.Fatal(err) + } + var replayed []uint64 + if _, err := Recover(region, func(txn *RecoveredTxn) error { + replayed = append(replayed, txn.Header.Seq) + return nil + }); err != nil { + t.Fatal(err) + } + return replayed +} + +func newTestJournal(t *testing.T) *Journal { + t.Helper() + dev, cleanup := createTestDevice(t, 10*1024*1024) + t.Cleanup(cleanup) + j, err := Init(dev, 0, DefaultJournalSlotSize, [16]byte{1}, 1, DefaultConfig()) + if err != nil { + t.Fatal(err) + } + return j +} + +func TestCheckpointKeepsIncompleteIntents(t *testing.T) { + j := newTestJournal(t) + seqs := commitIntents(t, j, true, false, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs[1:2]) { + t.Errorf("replayed %v after a checkpoint and a crash, want the incomplete intent %d", got, seqs[1]) + } +} + +func TestCheckpointPassesCompletedIntents(t *testing.T) { + j := newTestJournal(t) + seqs := commitIntents(t, j, true, false) + if err := j.CompleteIntent(seqs[1]); err != nil { + t.Fatal(err) + } + if done, err := j.Checkpoint(); err != nil || !done { + t.Fatalf("Checkpoint = %v, %v; want the head moved", done, err) + } + if got := replayedAfterCrash(t, j); len(got) > 0 { + t.Errorf("replayed %v after checkpointing completed intents", got) + } +} + +func TestStopLeavesIncompleteIntentsForReplay(t *testing.T) { + j := newTestJournal(t) + j.Start() + seqs := commitIntents(t, j, false) + if err := j.Stop(); err != nil { + t.Fatal(err) + } + if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs) { + t.Errorf("replayed %v after unmounting with an incomplete intent, want %v", got, seqs) + } +} From e45cbcb47f8aeb45cb64935fbe2c898165654c8f Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 00:36:28 -0600 Subject: [PATCH 02/10] fix(journal): keep the head behind intents whose operation has not completed Commit took a transaction out of activeTxns before the operation it logs ran, so the 5 s checkpoint, the flush-journal endpoint, a full journal or unmount could move the head past a committed intent that was not yet applied. A crash in that window lost the half-done operation. The journal now keeps each committed transaction pending until CompleteIntent. Checkpoints stop at the oldest active or pending transaction and sync the device first, so every change they pass is durable. Unmount leaves the journal dirty while an intent is pending. CompleteIntent is idempotent, and each operation defers it right after its commit, so an operation that fails after committing still closes its intent and is never replayed, as before. A Commit whose sync fails drops its intent, since its caller never runs the operation. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 17 +++++++ internal/journal/journal.go | 91 +++++++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 35 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a55cb6e9..00b48790 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -694,6 +694,13 @@ func (f *FS) Create(ctx context.Context, name string, flags uint32, mode uint32, return nil, nil, 0, syscall.EAGAIN } +// closeIntent completes an intent its operation left open by returning early, so a failed operation is never replayed. +func (f *FS) closeIntent(seq uint64) { + if err := f.journal.CompleteIntent(seq); err != nil { + log.Printf("[WARN] journal: completing intent %d failed: %v", seq, err) + } +} + // afterCreateCommit runs between a root create's commit and its first change; tests stop there as a crash would. var afterCreateCommit func() @@ -748,6 +755,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, if commitErr != nil { return nil, nil, 0, syscall.EIO } + defer f.closeIntent(intentSeq) if afterCreateCommit != nil { afterCreateCommit() } @@ -904,6 +912,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f if commitErr != nil { return nil, syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1039,6 +1048,7 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno { if commitErr != nil { return syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1132,6 +1142,7 @@ func (f *FS) Rmdir(ctx context.Context, name string) syscall.Errno { if commitErr != nil { return syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1504,6 +1515,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod log.Printf("[DEBUG] Create %q: journal commit failed: %v", name, commitErr) return nil, nil, 0, syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // Phase 3: Add directory entry FIRST (may acquire BlockBitmap lock for directory expansion) @@ -1715,6 +1727,7 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out * log.Printf("[DEBUG] Mkdir %q: journal commit failed: %v", name, commitErr) return nil, syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // Phase 3: Add directory entry in parent FIRST (may acquire BlockBitmap lock) @@ -1971,6 +1984,7 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall. if commitErr != nil { return syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // === End WAL transaction === @@ -2850,6 +2864,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: journal commit failed: %v", f.inodeNum, commitErr) return 0, syscall.EIO } + defer f.fs.closeIntent(intentSeq) txn = nil // Mark as committed so defer doesn't abort } // === End WAL transaction === @@ -3072,6 +3087,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { _ = f.fs.bitmapMgr.Free(ctx, newBlock) return commitErr } + defer f.fs.closeIntent(intentSeq) } ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: newBlock, Length: 1, Flags: extent.FlagAllocated} @@ -3220,6 +3236,7 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn if commitErr != nil { return syscall.EIO, nil } + defer f.fs.closeIntent(intentSeq) } // === End WAL transaction === diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 6e2a8128..9338bcef 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -99,6 +99,7 @@ type Journal struct { // Active transaction tracking activeTxns map[uint64]*ActiveTxn oldestActiveSeq uint64 + pendingIntents map[uint64]uint32 // committed but not yet completed, by seq, with where each starts in the ring // Checkpoint tracking lastCheckpointSeq uint64 @@ -143,6 +144,7 @@ func Open(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID generation: region.Superblock().NodeGeneration, config: config, activeTxns: make(map[uint64]*ActiveTxn), + pendingIntents: make(map[uint64]uint32), checkpointStop: make(chan struct{}), lastCheckpointSeq: region.Superblock().LastCheckpointSeq, } @@ -164,6 +166,7 @@ func Init(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID generation: generation, config: config, activeTxns: make(map[uint64]*ActiveTxn), + pendingIntents: make(map[uint64]uint32), checkpointStop: make(chan struct{}), lastCheckpointSeq: 0, } @@ -202,7 +205,7 @@ func (j *Journal) Stop() error { return } - if len(j.activeTxns) == 0 { + if len(j.activeTxns) == 0 && len(j.pendingIntents) == 0 { if err := j.region.MarkClean(); err != nil { stopErr = fmt.Errorf("failed to mark journal clean: %w", err) return @@ -237,26 +240,46 @@ func (j *Journal) periodicCheckpoint() { _, _ = j.Checkpoint() } -// Checkpoint advances the head past every committed transaction now instead -// of at the next interval. It reports false when there was nothing to do or -// a transaction is still active. +// Checkpoint advances the head now instead of at the next interval. It reports false when the head could not move. func (j *Journal) Checkpoint() (bool, error) { j.mu.Lock() defer j.mu.Unlock() + return j.advanceHead() +} - if len(j.activeTxns) > 0 { - return false, nil +// advanceHead moves the head up to the oldest transaction that is still active or whose operation has not completed. +// It syncs the device first, so the changes of every transaction it passes outlive their records. Must hold j.mu. +func (j *Journal) advanceHead() (bool, error) { + head, size := j.region.HeadOffset(), j.region.DataAreaSize() + after := func(pos uint32) uint32 { + if pos >= head { + return pos - head + } + return pos + size - head } - - tail := j.region.TailOffset() - if tail == j.region.HeadOffset() { + target, seq := j.region.TailOffset(), j.region.Superblock().NextSeq-1 + block := func(blockSeq uint64, pos uint32) { + if after(pos) < after(target) { + target = pos + } + seq = min(seq, blockSeq-1) + } + for blockSeq, active := range j.activeTxns { + block(blockSeq, active.tailPos) + } + for blockSeq, pos := range j.pendingIntents { + block(blockSeq, pos) + } + if target == head { return false, nil } - sb := j.region.Superblock() - if err := j.region.Checkpoint(sb.NextSeq-1, tail); err != nil { + if err := j.region.Sync(); err != nil { + return false, err + } + if err := j.region.Checkpoint(seq, target); err != nil { return false, err } - j.lastCheckpointSeq = sb.NextSeq - 1 + j.lastCheckpointSeq = seq j.lastCheckpointTime = time.Now() return true, nil } @@ -343,6 +366,7 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { } // Write to ring buffer + start := j.region.TailOffset() written, err := j.region.WriteAt(data) if err != nil { j.mu.Unlock() @@ -366,8 +390,9 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { j.commitCount++ j.bytesWritten += uint64(written) - // Remove from active set + // The operation runs after Commit, so the head must not pass this transaction until CompleteIntent. delete(j.activeTxns, seq) + j.pendingIntents[seq] = start // Update oldest active if seq == j.oldestActiveSeq { @@ -385,12 +410,14 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { case CommitSync: // Sync immediately for maximum durability if err := j.region.Sync(); err != nil { + j.dropIntent(seq) return 0, fmt.Errorf("failed to sync journal: %w", err) } case CommitBatch: // Batch mode: wait for periodic sync, reducing fsync frequency // This amortizes fsync cost across multiple commits if err := j.waitForBatchSync(); err != nil { + j.dropIntent(seq) return 0, fmt.Errorf("failed batch sync: %w", err) } case CommitAsync: @@ -400,6 +427,13 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { return seq, nil } +// dropIntent forgets an intent whose Commit failed, since its caller never runs the operation. +func (j *Journal) dropIntent(seq uint64) { + j.mu.Lock() + defer j.mu.Unlock() + delete(j.pendingIntents, seq) +} + // Abort abandons a transaction without committing. func (j *Journal) Abort(txn *Transaction) { j.mu.Lock() @@ -513,10 +547,17 @@ func (j *Journal) flushBatch() error { // If the system crashes between steps 2 and 4, recovery will replay the // operations. The idempotent execution ensures correctness regardless of // how many times operations are replayed. +// +// It is idempotent: once an intent is complete, later calls do nothing. func (j *Journal) CompleteIntent(intentSeq uint64) error { j.mu.Lock() defer j.mu.Unlock() + if _, ok := j.pendingIntents[intentSeq]; !ok { + return nil + } + delete(j.pendingIntents, intentSeq) + // Create completion record completion := NewIntentCompletion(intentSeq) data := completion.Serialize() @@ -575,28 +616,8 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { // forceCheckpoint tries to checkpoint to free journal space. // Must be called with lock held. func (j *Journal) forceCheckpoint() error { - // Can only checkpoint past committed transactions - if len(j.activeTxns) == 0 { - // No active txns - checkpoint everything - tail := j.region.TailOffset() - sb := j.region.Superblock() - return j.region.Checkpoint(sb.NextSeq-1, tail) - } - - // Find oldest active transaction position - var oldestPos uint32 - for _, active := range j.activeTxns { - if oldestPos == 0 || active.tailPos < oldestPos { - oldestPos = active.tailPos - } - } - - // Can only checkpoint up to just before oldest active - if oldestPos > 0 && oldestPos != j.region.HeadOffset() { - return j.region.Checkpoint(j.oldestActiveSeq-1, oldestPos) - } - - return nil + _, err := j.advanceHead() + return err } // updateOldestActive recalculates the oldest active sequence. From 28e4aeccb7487cdabdddc2e8231747e2d327e30c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 00:38:58 -0600 Subject: [PATCH 03/10] fix(fs): journal the file type with each create Both create paths logged the mode FUSE passed, while the runtime writes S_IFREG with the permission bits. A replayed create therefore depended on the caller's mode for the inode type, and fsck reported "invalid file type" when it lacked S_IFREG. Mkdir already logs S_IFDIR this way. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 00b48790..1ea30450 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -731,7 +731,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, txn = f.journal.Begin() if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode, + Mode: uint32(inode.S_IFREG) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(inode.InodeRoot), @@ -1485,7 +1485,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod // Log inode creation intent if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode, + Mode: uint32(inode.S_IFREG) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(d.inodeNum), From c0491242076ca453d31a68032e057ef041480bb9 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 00:41:09 -0600 Subject: [PATCH 04/10] test: track the checkpoint crash scenarios in the manifest Co-Authored-By: Claude Opus 5.5 --- tests/test-manifest.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 15b199a4..798672cd 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -707,6 +707,22 @@ scenarios: environments: go: internal/journal/journal_test.go::TestBatchCommitReturnsSyncError + - id: checkpoint-keeps-incomplete-intents + name: Checkpoints never pass an intent that has not completed + description: A transaction committed but not yet applied stays in the journal through a checkpoint and an unmount, and the next mount replays it; completed intents are checkpointed away + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestCheckpointKeepsIncompleteIntents + + - id: create-replays-after-checkpoint-crash + name: A create cut off after a checkpoint is replayed + description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean + priority: high + environments: + go: internal/goclusterfs/journal_crash_test.go::TestCreateReplaysAfterCheckpointAndCrash + docker: TODO # needs a pause between commit and completion, which only the Go test hook gives + vm: TODO # a power failure cannot be timed into the millisecond window between commit and completion without that hook + - id: intent-completion-serialize name: IntentCompletion serialization description: IntentCompletion record serializes and deserializes correctly From da1dd914410ef9333904de772db82971b8be4233 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 00:52:53 -0600 Subject: [PATCH 05/10] test(journal): complete intents before expecting a checkpoint or clean stop TestJournalCleanOnStop and TestCheckpointAdvancesHeadNow committed without CompleteIntent and expected the journal to drop the transaction, which is the bug. They now complete the intent first, and the checkpoint test also checks that the head does not move before completion. Co-Authored-By: Claude Opus 5.5 --- internal/journal/journal_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/internal/journal/journal_test.go b/internal/journal/journal_test.go index aa5f731f..4805da22 100644 --- a/internal/journal/journal_test.go +++ b/internal/journal/journal_test.go @@ -266,9 +266,13 @@ func TestJournalCleanOnStop(t *testing.T) { if err := txn.LogInodeCreate(testInodePayload(100)); err != nil { t.Fatal(err) } - if _, err := j1.Commit(txn); err != nil { + seq, err := j1.Commit(txn) + if err != nil { t.Fatalf("Commit failed: %v", err) } + if err := j1.CompleteIntent(seq); err != nil { + t.Fatal(err) + } // Stop should mark clean if err := j1.Stop(); err != nil { @@ -712,12 +716,19 @@ func TestCheckpointAdvancesHeadNow(t *testing.T) { if done, err := j.Checkpoint(); err != nil || done { t.Fatalf("active transaction: Checkpoint() = %v, %v; want false, nil", done, err) } - if _, err := j.Commit(txn); err != nil { + seq, err := j.Commit(txn) + if err != nil { + t.Fatal(err) + } + if done, err := j.Checkpoint(); err != nil || done { + t.Fatalf("committed, not completed: Checkpoint() = %v, %v; want false, nil", done, err) + } + if err := j.CompleteIntent(seq); err != nil { t.Fatal(err) } if done, err := j.Checkpoint(); err != nil || !done { - t.Fatalf("after commit: Checkpoint() = %v, %v; want true, nil", done, err) + t.Fatalf("after completion: Checkpoint() = %v, %v; want true, nil", done, err) } if head, tail := j.Region().HeadOffset(), j.Region().TailOffset(); head != tail { t.Fatalf("head %d != tail %d after Checkpoint", head, tail) From 162748f1e6ea5a8f0fdcac7fd30059b1c6352d3e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 01:10:14 -0600 Subject: [PATCH 06/10] test(fs): replay of an interrupted truncate keeps mapped blocks Truncate journals the logical block where the freed range starts, and replay frees physical blocks by that number. A crash between a truncate's commit and its changes then makes replay free blocks that files still map. The crash hook now covers setattr too. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 11 ++- internal/goclusterfs/journal_crash_test.go | 86 ++++++++++++++++------ 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 1ea30450..a8d450d4 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -701,8 +701,8 @@ func (f *FS) closeIntent(seq uint64) { } } -// afterCreateCommit runs between a root create's commit and its first change; tests stop there as a crash would. -var afterCreateCommit func() +// afterCommit runs between an operation's journal commit and its first change; tests stop there as a crash would. +var afterCommit func(op string) // createFileInternal implements file creation for the root directory. func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (*fs.Inode, fs.FileHandle, uint32, syscall.Errno) { @@ -756,8 +756,8 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, return nil, nil, 0, syscall.EIO } defer f.closeIntent(intentSeq) - if afterCreateCommit != nil { - afterCreateCommit() + if afterCommit != nil { + afterCommit("create") } } // === End WAL transaction - intent is now durable === @@ -3237,6 +3237,9 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn return syscall.EIO, nil } defer f.fs.closeIntent(intentSeq) + if afterCommit != nil { + afterCommit("setattr") + } } // === End WAL transaction === diff --git a/internal/goclusterfs/journal_crash_test.go b/internal/goclusterfs/journal_crash_test.go index f25cd0b7..98f5a127 100644 --- a/internal/goclusterfs/journal_crash_test.go +++ b/internal/goclusterfs/journal_crash_test.go @@ -11,49 +11,89 @@ import ( "github.com/supporttools/GoClusterFS/internal/superblock" ) -// A checkpoint between a create's commit and its changes, then a crash: the next mount must still replay the create. -func TestCreateReplaysAfterCheckpointAndCrash(t *testing.T) { - path := newFormattedImage(t) - fsys := openFormattedFS(t, path) - ctx := context.Background() - if err := fsys.InitRoot(ctx); err != nil { +// journaledFS opens a formatted image with a root directory and a journal in slot 0's region. +func journaledFS(t *testing.T) (fsys *FS, path string, offset, size uint64) { + t.Helper() + path = newFormattedImage(t) + fsys = openFormattedFS(t, path) + if err := fsys.InitRoot(context.Background()); err != nil { t.Fatal(err) } sb, err := superblock.Read(fsys.dev) if err != nil { t.Fatal(err) } - offset, size := sb.JournalRegionOffset, sb.JournalSlotSize() - j, err := journal.Init(fsys.dev, offset, size, fsys.nodeUUID, 1, journal.DefaultConfig()) - if err != nil { + offset, size = sb.JournalRegionOffset, sb.JournalSlotSize() + if fsys.journal, err = journal.Init(fsys.dev, offset, size, fsys.nodeUUID, 1, journal.DefaultConfig()); err != nil { t.Fatal(err) } - fsys.journal = j + return fsys, path, offset, size +} +// crashAfterCommit makes the next op stop after its commit, as a crash would, once a checkpoint has run. +func crashAfterCommit(t *testing.T, fsys *FS, op string) <-chan struct{} { + t.Helper() crashed := make(chan struct{}) - afterCreateCommit = func() { - if _, err := j.Checkpoint(); err != nil { + afterCommit = func(got string) { + if got != op { + return + } + if _, err := fsys.journal.Checkpoint(); err != nil { t.Errorf("checkpoint: %v", err) } close(crashed) select {} } - t.Cleanup(func() { afterCreateCommit = nil }) - go fsys.Create(ctx, "crashed.txt", 0, 0o644, &fuse.EntryOut{}) - <-crashed + t.Cleanup(func() { afterCommit = nil }) + return crashed +} - remounted := openFormattedFS(t, path) +// remountAndReplay opens the image as the next mount would and replays the journal. +func remountAndReplay(t *testing.T, path string, offset, size uint64) *FS { + t.Helper() + fsys := openFormattedFS(t, path) replay := journal.CreateTxnRecoveryHandler(&journal.RecoveryContext{ - Dev: remounted.dev, InodeTbl: remounted.inodeTbl, BitmapMgr: remounted.bitmapMgr, - DirMgr: remounted.dirMgr, ExtentMgr: remounted.extentMgr, + Dev: fsys.dev, InodeTbl: fsys.inodeTbl, BitmapMgr: fsys.bitmapMgr, + DirMgr: fsys.dirMgr, ExtentMgr: fsys.extentMgr, }) - result, err := journal.ScanAndRecover(remounted.dev, offset, size, replay) - if err != nil { + if _, err := journal.ScanAndRecover(fsys.dev, offset, size, replay); err != nil { t.Fatalf("replay: %v", err) } - t.Logf("replay: %+v", result) - lookupFile(t, remounted, "crashed.txt") - if r := runFsck(t, remounted, fsck.DefaultOptions()); len(r.Problems) > 0 { + return fsys +} + +func expectCleanFsck(t *testing.T, fsys *FS) { + t.Helper() + if r := runFsck(t, fsys, fsck.DefaultOptions()); len(r.Problems) > 0 { t.Fatalf("fsck after replay:%s", problemList(r)) } } + +// A checkpoint between a create's commit and its changes, then a crash: the next mount must still replay the create. +func TestCreateReplaysAfterCheckpointAndCrash(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + crashed := crashAfterCommit(t, fsys, "create") + go fsys.Create(context.Background(), "crashed.txt", 0, 0o644, &fuse.EntryOut{}) + <-crashed + + remounted := remountAndReplay(t, path, offset, size) + lookupFile(t, remounted, "crashed.txt") + expectCleanFsck(t, remounted) +} + +// A crash after a truncate's commit: replay must not free blocks that files still map. +func TestTruncateReplayKeepsMappedBlocks(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + victim := createFile(t, fsys, "victim.bin") + writeAt(t, victim, pattern(1, 3*BlockSize), 0) + other := createFile(t, fsys, "other.bin") + writeAt(t, other, pattern(2, 2*BlockSize), 0) + + crashed := crashAfterCommit(t, fsys, "setattr") + go tryTruncate(victim, BlockSize) + <-crashed + + remounted := remountAndReplay(t, path, offset, size) + expectCleanFsck(t, remounted) + checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, 2*BlockSize)) +} From 45dce8957a479b076e21bed5e2bdd3964b5d8582 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 01:11:06 -0600 Subject: [PATCH 07/10] fix(journal): never free blocks by number during replay Truncate and unlink log the logical file blocks they free, and replay freed physical blocks with those numbers. Replaying an interrupted truncate or unlink in a subdirectory therefore freed blocks files still map, starting at block 0, which the root directory uses. Replay now frees nothing for a BlockFree record: an interrupted free leaves its blocks allocated, which fsck reports, instead of freeing blocks in use. Co-Authored-By: Claude Opus 5.5 --- internal/journal/cross_node_recovery.go | 38 +++------------- internal/journal/cross_node_recovery_test.go | 47 ++++---------------- tests/test-manifest.yaml | 9 ++++ 3 files changed, 22 insertions(+), 72 deletions(-) diff --git a/internal/journal/cross_node_recovery.go b/internal/journal/cross_node_recovery.go index aae28eae..e7c4d6e5 100644 --- a/internal/journal/cross_node_recovery.go +++ b/internal/journal/cross_node_recovery.go @@ -393,44 +393,16 @@ func applyBlockAlloc(ctx context.Context, rctx *RecoveryContext, payload []byte) return applied, nil } -// applyBlockFree marks blocks as free if currently allocated. +// applyBlockFree frees nothing: truncate and unlink log logical file blocks, and freeing those numbers as physical +// blocks would free blocks other files map. An interrupted free leaves its blocks allocated, which fsck reports. func applyBlockFree(ctx context.Context, rctx *RecoveryContext, payload []byte) (bool, error) { p, err := DeserializeBlockFree(payload) if err != nil { return false, err } - - applied := false - for i := uint32(0); i < p.Count; i++ { - blockNum := p.StartBlock + uint64(i) - - // Check if allocated - isAllocated, err := rctx.BitmapMgr.IsAllocated(blockNum) - if err != nil { - return applied, fmt.Errorf("failed to check block %d: %w", blockNum, err) - } - - if !isAllocated { - // Already free - idempotent skip for this block - continue - } - - // Free the block - if err := rctx.BitmapMgr.Free(ctx, blockNum); err != nil { - return applied, fmt.Errorf("failed to free block %d: %w", blockNum, err) - } - applied = true - } - - if applied { - log.Printf("[Recovery] BlockFree: freed blocks %d-%d for inode %d", - p.StartBlock, p.StartBlock+uint64(p.Count)-1, p.InodeNum) - } else { - log.Printf("[Recovery] BlockFree: blocks %d-%d already free, skipping", - p.StartBlock, p.StartBlock+uint64(p.Count)-1) - } - - return applied, nil + log.Printf("[Recovery] BlockFree: leaving file blocks %d-%d of inode %d allocated", + p.StartBlock, p.StartBlock+uint64(p.Count)-1, p.InodeNum) + return false, nil } // applyExtentAdd adds an extent to an inode if not already present. diff --git a/internal/journal/cross_node_recovery_test.go b/internal/journal/cross_node_recovery_test.go index b29a1a3c..b18352e6 100644 --- a/internal/journal/cross_node_recovery_test.go +++ b/internal/journal/cross_node_recovery_test.go @@ -252,53 +252,22 @@ func TestApplyBlockAlloc(t *testing.T) { } } -// TestApplyBlockFree tests freeing a block during recovery. -func TestApplyBlockFree(t *testing.T) { +// Replay of a free leaves the block allocated: the payload names a file block, and that number may be another file's. +func TestApplyBlockFreeLeavesBlocksAllocated(t *testing.T) { rctx, cleanup := testRecoveryContext(t) defer cleanup() ctx := context.Background() - // First allocate a block - allocPayload := &BlockAllocPayload{ - InodeNum: 100, - StartBlock: 6, - Count: 1, - Purpose: BlockPurposeData, - } + allocPayload := &BlockAllocPayload{InodeNum: 100, StartBlock: 6, Count: 1, Purpose: BlockPurposeData} if _, err := applyBlockAlloc(ctx, rctx, allocPayload.Serialize()); err != nil { t.Fatalf("Setup: applyBlockAlloc failed: %v", err) } - - // Free the block - freePayload := &BlockFreePayload{ - InodeNum: 100, - StartBlock: 6, - Count: 1, + freePayload := &BlockFreePayload{InodeNum: 100, StartBlock: 6, Count: 1} + if applied, err := applyBlockFree(ctx, rctx, freePayload.Serialize()); err != nil || applied { + t.Fatalf("applyBlockFree = %v, %v; want false, nil", applied, err) } - applied, err := applyBlockFree(ctx, rctx, freePayload.Serialize()) - if err != nil { - t.Fatalf("applyBlockFree failed: %v", err) - } - if !applied { - t.Error("applyBlockFree returned false, expected true") - } - - // Verify the block is free - isAlloc, err := rctx.BitmapMgr.IsAllocated(freePayload.StartBlock) - if err != nil { - t.Fatalf("Failed to check block allocation: %v", err) - } - if isAlloc { - t.Error("Block should be marked as free") - } - - // Test idempotency - freeing again should skip - applied, err = applyBlockFree(ctx, rctx, freePayload.Serialize()) - if err != nil { - t.Fatalf("applyBlockFree (idempotent) failed: %v", err) - } - if applied { - t.Error("applyBlockFree returned true on second call, expected false (idempotent)") + if isAlloc, err := rctx.BitmapMgr.IsAllocated(6); err != nil || !isAlloc { + t.Errorf("block 6 allocated = %v, %v; want it left allocated", isAlloc, err) } } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 798672cd..95460ad4 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -723,6 +723,15 @@ scenarios: docker: TODO # needs a pause between commit and completion, which only the Go test hook gives vm: TODO # a power failure cannot be timed into the millisecond window between commit and completion without that hook + - id: truncate-replay-keeps-mapped-blocks + name: Replay of an interrupted truncate keeps blocks files still map + description: A truncate stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount frees no block a file maps, and fsck is clean + priority: high + environments: + go: internal/goclusterfs/journal_crash_test.go::TestTruncateReplayKeepsMappedBlocks + docker: TODO # needs a pause between commit and completion, which only the Go test hook gives + vm: TODO # a power failure cannot be timed into the millisecond window between commit and completion without that hook + - id: intent-completion-serialize name: IntentCompletion serialization description: IntentCompletion record serializes and deserializes correctly From 00d60754a13697519980adb5fab8f31585bf9182 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 01:20:26 -0600 Subject: [PATCH 08/10] fix(journal): keep one sector free so a full ring never reads as empty Records are sector-aligned and WriteAt accepted one that used the last free byte, so a ring filled while a pending intent pinned the head ended with head == tail, which recovery reads as empty. The next write then treated the whole ring as free and overwrote the pending intent. AvailableSpace now keeps one sector free, and Commit and CompleteIntent check the sector-aligned size WriteAt writes before deciding to checkpoint. Recovery also never finished when the tail sat at the end of the data area: the scan wrapped to 0 and kept looking for it. It now stops there. Mkdir in the root now journals only the permission bits with S_IFDIR, as mkdir in subdirectories does. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 2 +- internal/journal/checkpoint_test.go | 66 +++++++++++++++++++++++++++++ internal/journal/journal.go | 17 +++++--- internal/journal/recovery.go | 8 ++-- internal/journal/superblock.go | 12 ++---- internal/journal/superblock_test.go | 17 +++++--- 6 files changed, 96 insertions(+), 26 deletions(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a8d450d4..80bac9b2 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -888,7 +888,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f txn = f.journal.Begin() if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode | uint32(inode.S_IFDIR), + Mode: uint32(inode.S_IFDIR) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(inode.InodeRoot), diff --git a/internal/journal/checkpoint_test.go b/internal/journal/checkpoint_test.go index dd059be1..948c42fe 100644 --- a/internal/journal/checkpoint_test.go +++ b/internal/journal/checkpoint_test.go @@ -1,8 +1,10 @@ package journal import ( + "errors" "slices" "testing" + "time" ) // commitIntents commits one transaction per entry and completes those marked true. @@ -92,3 +94,67 @@ func TestStopLeavesIncompleteIntentsForReplay(t *testing.T) { t.Errorf("replayed %v after unmounting with an incomplete intent, want %v", got, seqs) } } + +// With the head pinned by a pending intent, other operations fill the ring until the journal is full, never past the intent. +func TestPendingIntentSurvivesFullRing(t *testing.T) { + j := newTestJournal(t) + commitIntents(t, j, slices.Repeat([]bool{true}, 100)...) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + pinned := commitIntents(t, j, false)[0] + for n := 1; ; n++ { + txn := j.Begin() + if err := txn.LogInodeCreate(testInodePayload(uint64(n))); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if errors.Is(err, ErrJournalFull) { + break + } + if err != nil { + t.Fatalf("op %d: %v", n, err) + } + if err := j.CompleteIntent(seq); err != nil && !errors.Is(err, ErrJournalFull) { + t.Fatalf("op %d: %v", n, err) + } + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + if head, tail := j.region.HeadOffset(), j.region.TailOffset(); head == tail { + t.Fatalf("op %d: head == tail == %d with intent %d pending, which reads as an empty journal", n, head, pinned) + } + } + if got := replayedAfterCrash(t, j); !slices.Contains(got, pinned) { + t.Errorf("a full journal replayed %v, want the pending intent %d", got, pinned) + } +} + +// A crash with the tail at the very end of the data area must not hang the next mount's scan. +func TestRecoveryEndsWithTailAtEnd(t *testing.T) { + dev, cleanup := createTestDevice(t, 1024*1024) + t.Cleanup(cleanup) + j, err := Init(dev, 0, uint64(JournalDataOffset)+3*4096, [16]byte{1}, 1, DefaultConfig()) + if err != nil { + t.Fatal(err) + } + commitIntents(t, j, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + pending := commitIntents(t, j, false) + if tail := j.region.TailOffset(); tail != j.region.DataAreaSize() { + t.Fatalf("tail %d, want the end of the %d-byte data area", tail, j.region.DataAreaSize()) + } + + done := make(chan []uint64, 1) + go func() { done <- replayedAfterCrash(t, j) }() + select { + case got := <-done: + if !slices.Equal(got, pending) { + t.Errorf("replayed %v, want %v", got, pending) + } + case <-time.After(10 * time.Second): + t.Fatal("recovery scan still running after 10s with the tail at the end of the data area") + } +} diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 9338bcef..ca2ec981 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -342,16 +342,17 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { return 0, fmt.Errorf("failed to serialize transaction: %w", err) } - // Check space + // Check space for the sector-aligned record WriteAt writes + need := sectorAligned(uint32(len(data))) available := j.region.AvailableSpace() - if uint32(len(data)) > available { + if need > available { // Try to force checkpoint to free space if err := j.forceCheckpoint(); err != nil { j.mu.Unlock() return 0, ErrJournalFull } available = j.region.AvailableSpace() - if uint32(len(data)) > available { + if need > available { j.mu.Unlock() return 0, ErrJournalFull } @@ -427,6 +428,10 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { return seq, nil } +func sectorAligned(n uint32) uint32 { + return (n + storage.SectorSize - 1) / storage.SectorSize * storage.SectorSize +} + // dropIntent forgets an intent whose Commit failed, since its caller never runs the operation. func (j *Journal) dropIntent(seq uint64) { j.mu.Lock() @@ -567,14 +572,14 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { alignedData := make([]byte, alignedSize) copy(alignedData, data) - // Check space + // Check space for the sector-aligned record WriteAt writes available := j.region.AvailableSpace() - if uint32(alignedSize) > available { + if sectorAligned(uint32(alignedSize)) > available { if err := j.forceCheckpoint(); err != nil { return ErrJournalFull } available = j.region.AvailableSpace() - if uint32(alignedSize) > available { + if sectorAligned(uint32(alignedSize)) > available { return ErrJournalFull } } diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index 43779080..d4a0bc8e 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -160,7 +160,7 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet } // Handle wrap-around - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } } @@ -238,7 +238,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Skip IntentCompletion records (already processed in Phase 1) if magic == IntentCompletionMagic { pos += storage.SectorSize - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } continue @@ -297,7 +297,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize pos = pos + alignedLen stats.BytesScanned += alignedLen - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } continue @@ -348,7 +348,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, stats.BytesScanned += alignedLen // Handle wrap-around - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } } diff --git a/internal/journal/superblock.go b/internal/journal/superblock.go index 77f6a1bc..9d4d175d 100644 --- a/internal/journal/superblock.go +++ b/internal/journal/superblock.go @@ -281,17 +281,13 @@ func (sb *JournalSuperblock) IsEmpty() bool { return sb.HeadOffset == sb.TailOffset && sb.WrapCount == 0 } -// AvailableSpace returns the number of bytes available for new transactions. +// AvailableSpace returns the bytes new records may use. One sector stays free, because a full ring would end at the head and read as empty. func (sb *JournalSuperblock) AvailableSpace() uint32 { + free := sb.HeadOffset - sb.TailOffset if sb.TailOffset >= sb.HeadOffset { - // Normal case: head is before tail - // Available = total - (tail - head) - used := sb.TailOffset - sb.HeadOffset - return sb.DataAreaSize - used + free = sb.DataAreaSize - (sb.TailOffset - sb.HeadOffset) } - // Wrapped case: tail is before head - // Available = head - tail - return sb.HeadOffset - sb.TailOffset + return free - min(free, storage.SectorSize) } // NeedsRecovery returns true if the journal needs to be replayed. diff --git a/internal/journal/superblock_test.go b/internal/journal/superblock_test.go index be1a70b1..ba2e69bf 100644 --- a/internal/journal/superblock_test.go +++ b/internal/journal/superblock_test.go @@ -2,6 +2,7 @@ package journal import ( "bytes" + "github.com/supporttools/GoClusterFS/internal/storage" "testing" ) @@ -188,7 +189,8 @@ func TestJournalSuperblockIsEmpty(t *testing.T) { } func TestJournalSuperblockAvailableSpace(t *testing.T) { - dataAreaSize := uint32(1000) + const sector = storage.SectorSize + dataAreaSize := uint32(10 * sector) tests := []struct { name string @@ -196,12 +198,13 @@ func TestJournalSuperblockAvailableSpace(t *testing.T) { tail uint32 wantSpace uint32 }{ - {"empty", 0, 0, 1000}, - {"partial used", 0, 300, 700}, - {"half used", 0, 500, 500}, - {"nearly full", 0, 900, 100}, - {"wrapped partial", 500, 200, 300}, // tail wrapped around - {"wrapped more", 800, 100, 700}, + {"empty", 0, 0, 9 * sector}, + {"partial used", 0, 3 * sector, 6 * sector}, + {"half used", 0, 5 * sector, 4 * sector}, + {"one sector free", 0, 9 * sector, 0}, + {"tail at the end", 2 * sector, 10 * sector, 1 * sector}, + {"wrapped partial", 5 * sector, 2 * sector, 2 * sector}, + {"wrapped full", 8 * sector, 7 * sector, 0}, } for _, tt := range tests { From 36d0722258b38578a5cd9c860eab36cd38d77ba8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 01:31:05 -0600 Subject: [PATCH 09/10] test(journal): expect the free sector in region space checks Co-Authored-By: Claude Opus 5.5 --- internal/journal/region_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/journal/region_test.go b/internal/journal/region_test.go index e1c4c071..677b3fe4 100644 --- a/internal/journal/region_test.go +++ b/internal/journal/region_test.go @@ -314,7 +314,7 @@ func TestRegionCheckpoint(t *testing.T) { // Total data area minus (tail - new head) // Note: tail is sector-aligned (SectorSize) not 1000 alignedTail := uint32(storage.SectorSize) // 1000 bytes rounds up to SectorSize - expected := region.DataAreaSize() - (alignedTail - 500) + expected := region.DataAreaSize() - (alignedTail - 500) - storage.SectorSize if space != expected { t.Errorf("Available space = %d, want %d", space, expected) } @@ -379,9 +379,9 @@ func TestRegionAvailableSpace(t *testing.T) { dataAreaSize := region.DataAreaSize() initialSpace := region.AvailableSpace() - // Initially should have full data area available - if initialSpace != dataAreaSize { - t.Errorf("Initial space = %d, want %d", initialSpace, dataAreaSize) + // Initially the whole data area but the one sector that stays free + if initialSpace != dataAreaSize-storage.SectorSize { + t.Errorf("Initial space = %d, want %d", initialSpace, dataAreaSize-storage.SectorSize) } // Write 1000 bytes @@ -393,8 +393,8 @@ func TestRegionAvailableSpace(t *testing.T) { // Space should decrease by sector-aligned amount (SectorSize, not 1000) afterWrite := region.AvailableSpace() alignedLen := uint32(storage.SectorSize) // 1000 bytes rounds up to SectorSize for O_DIRECT - if afterWrite != dataAreaSize-alignedLen { - t.Errorf("Space after write = %d, want %d", afterWrite, dataAreaSize-alignedLen) + if afterWrite != initialSpace-alignedLen { + t.Errorf("Space after write = %d, want %d", afterWrite, initialSpace-alignedLen) } } From cdb24a4181da241e467da52dad1565f5af0c7878 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Thu, 12 Nov 2026 01:31:25 -0600 Subject: [PATCH 10/10] test: track the full-ring and tail-at-end journal tests in the manifest Co-Authored-By: Claude Opus 5.5 --- tests/test-manifest.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 95460ad4..04fdfa99 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -714,6 +714,20 @@ scenarios: environments: go: internal/journal/checkpoint_test.go::TestCheckpointKeepsIncompleteIntents + - id: pending-intent-survives-full-ring + name: A full journal never overwrites a pending intent + description: While one intent pins the head, other operations fill the ring until Commit returns ErrJournalFull; the ring never ends at the head, where it would read as empty, and a crash still replays the pending intent + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestPendingIntentSurvivesFullRing + + - id: recovery-ends-with-tail-at-end + name: Recovery ends when the tail is at the end of the data area + description: A crash with the journal tail exactly at the end of the data area replays the pending transaction and returns, instead of scanning forever + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestRecoveryEndsWithTailAtEnd + - id: create-replays-after-checkpoint-crash name: A create cut off after a checkpoint is replayed description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean