diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a1db6fa4..80bac9b2 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -694,6 +694,16 @@ func (f *FS) Create(ctx context.Context, name string, flags uint32, mode uint32, return nil, nil, 0, syscall.EAGAIN } +// closeIntent completes an intent its operation left open by returning early, so a failed operation is never replayed. +func (f *FS) closeIntent(seq uint64) { + if err := f.journal.CompleteIntent(seq); err != nil { + log.Printf("[WARN] journal: completing intent %d failed: %v", seq, err) + } +} + +// afterCommit runs between an operation's journal commit and its first change; tests stop there as a crash would. +var afterCommit func(op string) + // createFileInternal implements file creation for the root directory. func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (*fs.Inode, fs.FileHandle, uint32, syscall.Errno) { inodeNum, err := f.inodeTbl.FindFreeSlot() @@ -721,7 +731,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, txn = f.journal.Begin() if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode, + Mode: uint32(inode.S_IFREG) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(inode.InodeRoot), @@ -745,6 +755,10 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, if commitErr != nil { return nil, nil, 0, syscall.EIO } + defer f.closeIntent(intentSeq) + if afterCommit != nil { + afterCommit("create") + } } // === End WAL transaction - intent is now durable === @@ -874,7 +888,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f txn = f.journal.Begin() if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode | uint32(inode.S_IFDIR), + Mode: uint32(inode.S_IFDIR) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(inode.InodeRoot), @@ -898,6 +912,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f if commitErr != nil { return nil, syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1033,6 +1048,7 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno { if commitErr != nil { return syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1126,6 +1142,7 @@ func (f *FS) Rmdir(ctx context.Context, name string) syscall.Errno { if commitErr != nil { return syscall.EIO } + defer f.closeIntent(intentSeq) } // === End WAL transaction - intent is now durable === @@ -1468,7 +1485,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod // Log inode creation intent if err := txn.LogInodeCreate(&journal.InodeCreatePayload{ InodeNum: uint64(inodeNum), - Mode: mode, + Mode: uint32(inode.S_IFREG) | (mode & 0777), UID: uid, GID: gid, ParentInodeNum: uint64(d.inodeNum), @@ -1498,6 +1515,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod log.Printf("[DEBUG] Create %q: journal commit failed: %v", name, commitErr) return nil, nil, 0, syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // Phase 3: Add directory entry FIRST (may acquire BlockBitmap lock for directory expansion) @@ -1709,6 +1727,7 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out * log.Printf("[DEBUG] Mkdir %q: journal commit failed: %v", name, commitErr) return nil, syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // Phase 3: Add directory entry in parent FIRST (may acquire BlockBitmap lock) @@ -1965,6 +1984,7 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall. if commitErr != nil { return syscall.EIO } + defer d.fs.closeIntent(intentSeq) } // === End WAL transaction === @@ -2844,6 +2864,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i log.Printf("[DEBUG] Write inode %d: journal commit failed: %v", f.inodeNum, commitErr) return 0, syscall.EIO } + defer f.fs.closeIntent(intentSeq) txn = nil // Mark as committed so defer doesn't abort } // === End WAL transaction === @@ -3066,6 +3087,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error { _ = f.fs.bitmapMgr.Free(ctx, newBlock) return commitErr } + defer f.fs.closeIntent(intentSeq) } ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: newBlock, Length: 1, Flags: extent.FlagAllocated} @@ -3214,6 +3236,10 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn if commitErr != nil { return syscall.EIO, nil } + defer f.fs.closeIntent(intentSeq) + if afterCommit != nil { + afterCommit("setattr") + } } // === End WAL transaction === diff --git a/internal/goclusterfs/journal_crash_test.go b/internal/goclusterfs/journal_crash_test.go new file mode 100644 index 00000000..98f5a127 --- /dev/null +++ b/internal/goclusterfs/journal_crash_test.go @@ -0,0 +1,99 @@ +package goclusterfs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +// journaledFS opens a formatted image with a root directory and a journal in slot 0's region. +func journaledFS(t *testing.T) (fsys *FS, path string, offset, size uint64) { + t.Helper() + path = newFormattedImage(t) + fsys = openFormattedFS(t, path) + if err := fsys.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + offset, size = sb.JournalRegionOffset, sb.JournalSlotSize() + if fsys.journal, err = journal.Init(fsys.dev, offset, size, fsys.nodeUUID, 1, journal.DefaultConfig()); err != nil { + t.Fatal(err) + } + return fsys, path, offset, size +} + +// crashAfterCommit makes the next op stop after its commit, as a crash would, once a checkpoint has run. +func crashAfterCommit(t *testing.T, fsys *FS, op string) <-chan struct{} { + t.Helper() + crashed := make(chan struct{}) + afterCommit = func(got string) { + if got != op { + return + } + if _, err := fsys.journal.Checkpoint(); err != nil { + t.Errorf("checkpoint: %v", err) + } + close(crashed) + select {} + } + t.Cleanup(func() { afterCommit = nil }) + return crashed +} + +// remountAndReplay opens the image as the next mount would and replays the journal. +func remountAndReplay(t *testing.T, path string, offset, size uint64) *FS { + t.Helper() + fsys := openFormattedFS(t, path) + replay := journal.CreateTxnRecoveryHandler(&journal.RecoveryContext{ + Dev: fsys.dev, InodeTbl: fsys.inodeTbl, BitmapMgr: fsys.bitmapMgr, + DirMgr: fsys.dirMgr, ExtentMgr: fsys.extentMgr, + }) + if _, err := journal.ScanAndRecover(fsys.dev, offset, size, replay); err != nil { + t.Fatalf("replay: %v", err) + } + return fsys +} + +func expectCleanFsck(t *testing.T, fsys *FS) { + t.Helper() + if r := runFsck(t, fsys, fsck.DefaultOptions()); len(r.Problems) > 0 { + t.Fatalf("fsck after replay:%s", problemList(r)) + } +} + +// A checkpoint between a create's commit and its changes, then a crash: the next mount must still replay the create. +func TestCreateReplaysAfterCheckpointAndCrash(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + crashed := crashAfterCommit(t, fsys, "create") + go fsys.Create(context.Background(), "crashed.txt", 0, 0o644, &fuse.EntryOut{}) + <-crashed + + remounted := remountAndReplay(t, path, offset, size) + lookupFile(t, remounted, "crashed.txt") + expectCleanFsck(t, remounted) +} + +// A crash after a truncate's commit: replay must not free blocks that files still map. +func TestTruncateReplayKeepsMappedBlocks(t *testing.T) { + fsys, path, offset, size := journaledFS(t) + victim := createFile(t, fsys, "victim.bin") + writeAt(t, victim, pattern(1, 3*BlockSize), 0) + other := createFile(t, fsys, "other.bin") + writeAt(t, other, pattern(2, 2*BlockSize), 0) + + crashed := crashAfterCommit(t, fsys, "setattr") + go tryTruncate(victim, BlockSize) + <-crashed + + remounted := remountAndReplay(t, path, offset, size) + expectCleanFsck(t, remounted) + checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, 2*BlockSize)) +} diff --git a/internal/journal/checkpoint_test.go b/internal/journal/checkpoint_test.go new file mode 100644 index 00000000..948c42fe --- /dev/null +++ b/internal/journal/checkpoint_test.go @@ -0,0 +1,160 @@ +package journal + +import ( + "errors" + "slices" + "testing" + "time" +) + +// commitIntents commits one transaction per entry and completes those marked true. +func commitIntents(t *testing.T, j *Journal, complete ...bool) []uint64 { + t.Helper() + var seqs []uint64 + for i, done := range complete { + txn := j.Begin() + if err := txn.LogInodeCreate(testInodePayload(uint64(100 + i))); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if err != nil { + t.Fatal(err) + } + if done { + if err := j.CompleteIntent(seq); err != nil { + t.Fatal(err) + } + } + seqs = append(seqs, seq) + } + return seqs +} + +// replayedAfterCrash reopens the region as the next mount would and returns the transactions it replays. +func replayedAfterCrash(t *testing.T, j *Journal) []uint64 { + t.Helper() + region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize) + if err != nil { + t.Fatal(err) + } + var replayed []uint64 + if _, err := Recover(region, func(txn *RecoveredTxn) error { + replayed = append(replayed, txn.Header.Seq) + return nil + }); err != nil { + t.Fatal(err) + } + return replayed +} + +func newTestJournal(t *testing.T) *Journal { + t.Helper() + dev, cleanup := createTestDevice(t, 10*1024*1024) + t.Cleanup(cleanup) + j, err := Init(dev, 0, DefaultJournalSlotSize, [16]byte{1}, 1, DefaultConfig()) + if err != nil { + t.Fatal(err) + } + return j +} + +func TestCheckpointKeepsIncompleteIntents(t *testing.T) { + j := newTestJournal(t) + seqs := commitIntents(t, j, true, false, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs[1:2]) { + t.Errorf("replayed %v after a checkpoint and a crash, want the incomplete intent %d", got, seqs[1]) + } +} + +func TestCheckpointPassesCompletedIntents(t *testing.T) { + j := newTestJournal(t) + seqs := commitIntents(t, j, true, false) + if err := j.CompleteIntent(seqs[1]); err != nil { + t.Fatal(err) + } + if done, err := j.Checkpoint(); err != nil || !done { + t.Fatalf("Checkpoint = %v, %v; want the head moved", done, err) + } + if got := replayedAfterCrash(t, j); len(got) > 0 { + t.Errorf("replayed %v after checkpointing completed intents", got) + } +} + +func TestStopLeavesIncompleteIntentsForReplay(t *testing.T) { + j := newTestJournal(t) + j.Start() + seqs := commitIntents(t, j, false) + if err := j.Stop(); err != nil { + t.Fatal(err) + } + if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs) { + t.Errorf("replayed %v after unmounting with an incomplete intent, want %v", got, seqs) + } +} + +// With the head pinned by a pending intent, other operations fill the ring until the journal is full, never past the intent. +func TestPendingIntentSurvivesFullRing(t *testing.T) { + j := newTestJournal(t) + commitIntents(t, j, slices.Repeat([]bool{true}, 100)...) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + pinned := commitIntents(t, j, false)[0] + for n := 1; ; n++ { + txn := j.Begin() + if err := txn.LogInodeCreate(testInodePayload(uint64(n))); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if errors.Is(err, ErrJournalFull) { + break + } + if err != nil { + t.Fatalf("op %d: %v", n, err) + } + if err := j.CompleteIntent(seq); err != nil && !errors.Is(err, ErrJournalFull) { + t.Fatalf("op %d: %v", n, err) + } + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + if head, tail := j.region.HeadOffset(), j.region.TailOffset(); head == tail { + t.Fatalf("op %d: head == tail == %d with intent %d pending, which reads as an empty journal", n, head, pinned) + } + } + if got := replayedAfterCrash(t, j); !slices.Contains(got, pinned) { + t.Errorf("a full journal replayed %v, want the pending intent %d", got, pinned) + } +} + +// A crash with the tail at the very end of the data area must not hang the next mount's scan. +func TestRecoveryEndsWithTailAtEnd(t *testing.T) { + dev, cleanup := createTestDevice(t, 1024*1024) + t.Cleanup(cleanup) + j, err := Init(dev, 0, uint64(JournalDataOffset)+3*4096, [16]byte{1}, 1, DefaultConfig()) + if err != nil { + t.Fatal(err) + } + commitIntents(t, j, true) + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + pending := commitIntents(t, j, false) + if tail := j.region.TailOffset(); tail != j.region.DataAreaSize() { + t.Fatalf("tail %d, want the end of the %d-byte data area", tail, j.region.DataAreaSize()) + } + + done := make(chan []uint64, 1) + go func() { done <- replayedAfterCrash(t, j) }() + select { + case got := <-done: + if !slices.Equal(got, pending) { + t.Errorf("replayed %v, want %v", got, pending) + } + case <-time.After(10 * time.Second): + t.Fatal("recovery scan still running after 10s with the tail at the end of the data area") + } +} diff --git a/internal/journal/cross_node_recovery.go b/internal/journal/cross_node_recovery.go index aae28eae..e7c4d6e5 100644 --- a/internal/journal/cross_node_recovery.go +++ b/internal/journal/cross_node_recovery.go @@ -393,44 +393,16 @@ func applyBlockAlloc(ctx context.Context, rctx *RecoveryContext, payload []byte) return applied, nil } -// applyBlockFree marks blocks as free if currently allocated. +// applyBlockFree frees nothing: truncate and unlink log logical file blocks, and freeing those numbers as physical +// blocks would free blocks other files map. An interrupted free leaves its blocks allocated, which fsck reports. func applyBlockFree(ctx context.Context, rctx *RecoveryContext, payload []byte) (bool, error) { p, err := DeserializeBlockFree(payload) if err != nil { return false, err } - - applied := false - for i := uint32(0); i < p.Count; i++ { - blockNum := p.StartBlock + uint64(i) - - // Check if allocated - isAllocated, err := rctx.BitmapMgr.IsAllocated(blockNum) - if err != nil { - return applied, fmt.Errorf("failed to check block %d: %w", blockNum, err) - } - - if !isAllocated { - // Already free - idempotent skip for this block - continue - } - - // Free the block - if err := rctx.BitmapMgr.Free(ctx, blockNum); err != nil { - return applied, fmt.Errorf("failed to free block %d: %w", blockNum, err) - } - applied = true - } - - if applied { - log.Printf("[Recovery] BlockFree: freed blocks %d-%d for inode %d", - p.StartBlock, p.StartBlock+uint64(p.Count)-1, p.InodeNum) - } else { - log.Printf("[Recovery] BlockFree: blocks %d-%d already free, skipping", - p.StartBlock, p.StartBlock+uint64(p.Count)-1) - } - - return applied, nil + log.Printf("[Recovery] BlockFree: leaving file blocks %d-%d of inode %d allocated", + p.StartBlock, p.StartBlock+uint64(p.Count)-1, p.InodeNum) + return false, nil } // applyExtentAdd adds an extent to an inode if not already present. diff --git a/internal/journal/cross_node_recovery_test.go b/internal/journal/cross_node_recovery_test.go index b29a1a3c..b18352e6 100644 --- a/internal/journal/cross_node_recovery_test.go +++ b/internal/journal/cross_node_recovery_test.go @@ -252,53 +252,22 @@ func TestApplyBlockAlloc(t *testing.T) { } } -// TestApplyBlockFree tests freeing a block during recovery. -func TestApplyBlockFree(t *testing.T) { +// Replay of a free leaves the block allocated: the payload names a file block, and that number may be another file's. +func TestApplyBlockFreeLeavesBlocksAllocated(t *testing.T) { rctx, cleanup := testRecoveryContext(t) defer cleanup() ctx := context.Background() - // First allocate a block - allocPayload := &BlockAllocPayload{ - InodeNum: 100, - StartBlock: 6, - Count: 1, - Purpose: BlockPurposeData, - } + allocPayload := &BlockAllocPayload{InodeNum: 100, StartBlock: 6, Count: 1, Purpose: BlockPurposeData} if _, err := applyBlockAlloc(ctx, rctx, allocPayload.Serialize()); err != nil { t.Fatalf("Setup: applyBlockAlloc failed: %v", err) } - - // Free the block - freePayload := &BlockFreePayload{ - InodeNum: 100, - StartBlock: 6, - Count: 1, + freePayload := &BlockFreePayload{InodeNum: 100, StartBlock: 6, Count: 1} + if applied, err := applyBlockFree(ctx, rctx, freePayload.Serialize()); err != nil || applied { + t.Fatalf("applyBlockFree = %v, %v; want false, nil", applied, err) } - applied, err := applyBlockFree(ctx, rctx, freePayload.Serialize()) - if err != nil { - t.Fatalf("applyBlockFree failed: %v", err) - } - if !applied { - t.Error("applyBlockFree returned false, expected true") - } - - // Verify the block is free - isAlloc, err := rctx.BitmapMgr.IsAllocated(freePayload.StartBlock) - if err != nil { - t.Fatalf("Failed to check block allocation: %v", err) - } - if isAlloc { - t.Error("Block should be marked as free") - } - - // Test idempotency - freeing again should skip - applied, err = applyBlockFree(ctx, rctx, freePayload.Serialize()) - if err != nil { - t.Fatalf("applyBlockFree (idempotent) failed: %v", err) - } - if applied { - t.Error("applyBlockFree returned true on second call, expected false (idempotent)") + if isAlloc, err := rctx.BitmapMgr.IsAllocated(6); err != nil || !isAlloc { + t.Errorf("block 6 allocated = %v, %v; want it left allocated", isAlloc, err) } } diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 6e2a8128..ca2ec981 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -99,6 +99,7 @@ type Journal struct { // Active transaction tracking activeTxns map[uint64]*ActiveTxn oldestActiveSeq uint64 + pendingIntents map[uint64]uint32 // committed but not yet completed, by seq, with where each starts in the ring // Checkpoint tracking lastCheckpointSeq uint64 @@ -143,6 +144,7 @@ func Open(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID generation: region.Superblock().NodeGeneration, config: config, activeTxns: make(map[uint64]*ActiveTxn), + pendingIntents: make(map[uint64]uint32), checkpointStop: make(chan struct{}), lastCheckpointSeq: region.Superblock().LastCheckpointSeq, } @@ -164,6 +166,7 @@ func Init(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID generation: generation, config: config, activeTxns: make(map[uint64]*ActiveTxn), + pendingIntents: make(map[uint64]uint32), checkpointStop: make(chan struct{}), lastCheckpointSeq: 0, } @@ -202,7 +205,7 @@ func (j *Journal) Stop() error { return } - if len(j.activeTxns) == 0 { + if len(j.activeTxns) == 0 && len(j.pendingIntents) == 0 { if err := j.region.MarkClean(); err != nil { stopErr = fmt.Errorf("failed to mark journal clean: %w", err) return @@ -237,26 +240,46 @@ func (j *Journal) periodicCheckpoint() { _, _ = j.Checkpoint() } -// Checkpoint advances the head past every committed transaction now instead -// of at the next interval. It reports false when there was nothing to do or -// a transaction is still active. +// Checkpoint advances the head now instead of at the next interval. It reports false when the head could not move. func (j *Journal) Checkpoint() (bool, error) { j.mu.Lock() defer j.mu.Unlock() + return j.advanceHead() +} - if len(j.activeTxns) > 0 { - return false, nil +// advanceHead moves the head up to the oldest transaction that is still active or whose operation has not completed. +// It syncs the device first, so the changes of every transaction it passes outlive their records. Must hold j.mu. +func (j *Journal) advanceHead() (bool, error) { + head, size := j.region.HeadOffset(), j.region.DataAreaSize() + after := func(pos uint32) uint32 { + if pos >= head { + return pos - head + } + return pos + size - head } - - tail := j.region.TailOffset() - if tail == j.region.HeadOffset() { + target, seq := j.region.TailOffset(), j.region.Superblock().NextSeq-1 + block := func(blockSeq uint64, pos uint32) { + if after(pos) < after(target) { + target = pos + } + seq = min(seq, blockSeq-1) + } + for blockSeq, active := range j.activeTxns { + block(blockSeq, active.tailPos) + } + for blockSeq, pos := range j.pendingIntents { + block(blockSeq, pos) + } + if target == head { return false, nil } - sb := j.region.Superblock() - if err := j.region.Checkpoint(sb.NextSeq-1, tail); err != nil { + if err := j.region.Sync(); err != nil { + return false, err + } + if err := j.region.Checkpoint(seq, target); err != nil { return false, err } - j.lastCheckpointSeq = sb.NextSeq - 1 + j.lastCheckpointSeq = seq j.lastCheckpointTime = time.Now() return true, nil } @@ -319,16 +342,17 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { return 0, fmt.Errorf("failed to serialize transaction: %w", err) } - // Check space + // Check space for the sector-aligned record WriteAt writes + need := sectorAligned(uint32(len(data))) available := j.region.AvailableSpace() - if uint32(len(data)) > available { + if need > available { // Try to force checkpoint to free space if err := j.forceCheckpoint(); err != nil { j.mu.Unlock() return 0, ErrJournalFull } available = j.region.AvailableSpace() - if uint32(len(data)) > available { + if need > available { j.mu.Unlock() return 0, ErrJournalFull } @@ -343,6 +367,7 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { } // Write to ring buffer + start := j.region.TailOffset() written, err := j.region.WriteAt(data) if err != nil { j.mu.Unlock() @@ -366,8 +391,9 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { j.commitCount++ j.bytesWritten += uint64(written) - // Remove from active set + // The operation runs after Commit, so the head must not pass this transaction until CompleteIntent. delete(j.activeTxns, seq) + j.pendingIntents[seq] = start // Update oldest active if seq == j.oldestActiveSeq { @@ -385,12 +411,14 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { case CommitSync: // Sync immediately for maximum durability if err := j.region.Sync(); err != nil { + j.dropIntent(seq) return 0, fmt.Errorf("failed to sync journal: %w", err) } case CommitBatch: // Batch mode: wait for periodic sync, reducing fsync frequency // This amortizes fsync cost across multiple commits if err := j.waitForBatchSync(); err != nil { + j.dropIntent(seq) return 0, fmt.Errorf("failed batch sync: %w", err) } case CommitAsync: @@ -400,6 +428,17 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { return seq, nil } +func sectorAligned(n uint32) uint32 { + return (n + storage.SectorSize - 1) / storage.SectorSize * storage.SectorSize +} + +// dropIntent forgets an intent whose Commit failed, since its caller never runs the operation. +func (j *Journal) dropIntent(seq uint64) { + j.mu.Lock() + defer j.mu.Unlock() + delete(j.pendingIntents, seq) +} + // Abort abandons a transaction without committing. func (j *Journal) Abort(txn *Transaction) { j.mu.Lock() @@ -513,10 +552,17 @@ func (j *Journal) flushBatch() error { // If the system crashes between steps 2 and 4, recovery will replay the // operations. The idempotent execution ensures correctness regardless of // how many times operations are replayed. +// +// It is idempotent: once an intent is complete, later calls do nothing. func (j *Journal) CompleteIntent(intentSeq uint64) error { j.mu.Lock() defer j.mu.Unlock() + if _, ok := j.pendingIntents[intentSeq]; !ok { + return nil + } + delete(j.pendingIntents, intentSeq) + // Create completion record completion := NewIntentCompletion(intentSeq) data := completion.Serialize() @@ -526,14 +572,14 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { alignedData := make([]byte, alignedSize) copy(alignedData, data) - // Check space + // Check space for the sector-aligned record WriteAt writes available := j.region.AvailableSpace() - if uint32(alignedSize) > available { + if sectorAligned(uint32(alignedSize)) > available { if err := j.forceCheckpoint(); err != nil { return ErrJournalFull } available = j.region.AvailableSpace() - if uint32(alignedSize) > available { + if sectorAligned(uint32(alignedSize)) > available { return ErrJournalFull } } @@ -575,28 +621,8 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { // forceCheckpoint tries to checkpoint to free journal space. // Must be called with lock held. func (j *Journal) forceCheckpoint() error { - // Can only checkpoint past committed transactions - if len(j.activeTxns) == 0 { - // No active txns - checkpoint everything - tail := j.region.TailOffset() - sb := j.region.Superblock() - return j.region.Checkpoint(sb.NextSeq-1, tail) - } - - // Find oldest active transaction position - var oldestPos uint32 - for _, active := range j.activeTxns { - if oldestPos == 0 || active.tailPos < oldestPos { - oldestPos = active.tailPos - } - } - - // Can only checkpoint up to just before oldest active - if oldestPos > 0 && oldestPos != j.region.HeadOffset() { - return j.region.Checkpoint(j.oldestActiveSeq-1, oldestPos) - } - - return nil + _, err := j.advanceHead() + return err } // updateOldestActive recalculates the oldest active sequence. diff --git a/internal/journal/journal_test.go b/internal/journal/journal_test.go index aa5f731f..4805da22 100644 --- a/internal/journal/journal_test.go +++ b/internal/journal/journal_test.go @@ -266,9 +266,13 @@ func TestJournalCleanOnStop(t *testing.T) { if err := txn.LogInodeCreate(testInodePayload(100)); err != nil { t.Fatal(err) } - if _, err := j1.Commit(txn); err != nil { + seq, err := j1.Commit(txn) + if err != nil { t.Fatalf("Commit failed: %v", err) } + if err := j1.CompleteIntent(seq); err != nil { + t.Fatal(err) + } // Stop should mark clean if err := j1.Stop(); err != nil { @@ -712,12 +716,19 @@ func TestCheckpointAdvancesHeadNow(t *testing.T) { if done, err := j.Checkpoint(); err != nil || done { t.Fatalf("active transaction: Checkpoint() = %v, %v; want false, nil", done, err) } - if _, err := j.Commit(txn); err != nil { + seq, err := j.Commit(txn) + if err != nil { + t.Fatal(err) + } + if done, err := j.Checkpoint(); err != nil || done { + t.Fatalf("committed, not completed: Checkpoint() = %v, %v; want false, nil", done, err) + } + if err := j.CompleteIntent(seq); err != nil { t.Fatal(err) } if done, err := j.Checkpoint(); err != nil || !done { - t.Fatalf("after commit: Checkpoint() = %v, %v; want true, nil", done, err) + t.Fatalf("after completion: Checkpoint() = %v, %v; want true, nil", done, err) } if head, tail := j.Region().HeadOffset(), j.Region().TailOffset(); head != tail { t.Fatalf("head %d != tail %d after Checkpoint", head, tail) diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index 43779080..d4a0bc8e 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -160,7 +160,7 @@ func scanForCompletions(region *Region, head, tail, dataAreaSize uint32, complet } // Handle wrap-around - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } } @@ -238,7 +238,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, // Skip IntentCompletion records (already processed in Phase 1) if magic == IntentCompletionMagic { pos += storage.SectorSize - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } continue @@ -297,7 +297,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, alignedLen := ((txnHeader.TotalLength + storage.SectorSize - 1) / storage.SectorSize) * storage.SectorSize pos = pos + alignedLen stats.BytesScanned += alignedLen - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } continue @@ -348,7 +348,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, stats.BytesScanned += alignedLen // Handle wrap-around - if pos >= dataAreaSize { + if pos >= dataAreaSize && pos != tail { pos = pos - dataAreaSize } } diff --git a/internal/journal/region_test.go b/internal/journal/region_test.go index e1c4c071..677b3fe4 100644 --- a/internal/journal/region_test.go +++ b/internal/journal/region_test.go @@ -314,7 +314,7 @@ func TestRegionCheckpoint(t *testing.T) { // Total data area minus (tail - new head) // Note: tail is sector-aligned (SectorSize) not 1000 alignedTail := uint32(storage.SectorSize) // 1000 bytes rounds up to SectorSize - expected := region.DataAreaSize() - (alignedTail - 500) + expected := region.DataAreaSize() - (alignedTail - 500) - storage.SectorSize if space != expected { t.Errorf("Available space = %d, want %d", space, expected) } @@ -379,9 +379,9 @@ func TestRegionAvailableSpace(t *testing.T) { dataAreaSize := region.DataAreaSize() initialSpace := region.AvailableSpace() - // Initially should have full data area available - if initialSpace != dataAreaSize { - t.Errorf("Initial space = %d, want %d", initialSpace, dataAreaSize) + // Initially the whole data area but the one sector that stays free + if initialSpace != dataAreaSize-storage.SectorSize { + t.Errorf("Initial space = %d, want %d", initialSpace, dataAreaSize-storage.SectorSize) } // Write 1000 bytes @@ -393,8 +393,8 @@ func TestRegionAvailableSpace(t *testing.T) { // Space should decrease by sector-aligned amount (SectorSize, not 1000) afterWrite := region.AvailableSpace() alignedLen := uint32(storage.SectorSize) // 1000 bytes rounds up to SectorSize for O_DIRECT - if afterWrite != dataAreaSize-alignedLen { - t.Errorf("Space after write = %d, want %d", afterWrite, dataAreaSize-alignedLen) + if afterWrite != initialSpace-alignedLen { + t.Errorf("Space after write = %d, want %d", afterWrite, initialSpace-alignedLen) } } diff --git a/internal/journal/superblock.go b/internal/journal/superblock.go index 77f6a1bc..9d4d175d 100644 --- a/internal/journal/superblock.go +++ b/internal/journal/superblock.go @@ -281,17 +281,13 @@ func (sb *JournalSuperblock) IsEmpty() bool { return sb.HeadOffset == sb.TailOffset && sb.WrapCount == 0 } -// AvailableSpace returns the number of bytes available for new transactions. +// AvailableSpace returns the bytes new records may use. One sector stays free, because a full ring would end at the head and read as empty. func (sb *JournalSuperblock) AvailableSpace() uint32 { + free := sb.HeadOffset - sb.TailOffset if sb.TailOffset >= sb.HeadOffset { - // Normal case: head is before tail - // Available = total - (tail - head) - used := sb.TailOffset - sb.HeadOffset - return sb.DataAreaSize - used + free = sb.DataAreaSize - (sb.TailOffset - sb.HeadOffset) } - // Wrapped case: tail is before head - // Available = head - tail - return sb.HeadOffset - sb.TailOffset + return free - min(free, storage.SectorSize) } // NeedsRecovery returns true if the journal needs to be replayed. diff --git a/internal/journal/superblock_test.go b/internal/journal/superblock_test.go index be1a70b1..ba2e69bf 100644 --- a/internal/journal/superblock_test.go +++ b/internal/journal/superblock_test.go @@ -2,6 +2,7 @@ package journal import ( "bytes" + "github.com/supporttools/GoClusterFS/internal/storage" "testing" ) @@ -188,7 +189,8 @@ func TestJournalSuperblockIsEmpty(t *testing.T) { } func TestJournalSuperblockAvailableSpace(t *testing.T) { - dataAreaSize := uint32(1000) + const sector = storage.SectorSize + dataAreaSize := uint32(10 * sector) tests := []struct { name string @@ -196,12 +198,13 @@ func TestJournalSuperblockAvailableSpace(t *testing.T) { tail uint32 wantSpace uint32 }{ - {"empty", 0, 0, 1000}, - {"partial used", 0, 300, 700}, - {"half used", 0, 500, 500}, - {"nearly full", 0, 900, 100}, - {"wrapped partial", 500, 200, 300}, // tail wrapped around - {"wrapped more", 800, 100, 700}, + {"empty", 0, 0, 9 * sector}, + {"partial used", 0, 3 * sector, 6 * sector}, + {"half used", 0, 5 * sector, 4 * sector}, + {"one sector free", 0, 9 * sector, 0}, + {"tail at the end", 2 * sector, 10 * sector, 1 * sector}, + {"wrapped partial", 5 * sector, 2 * sector, 2 * sector}, + {"wrapped full", 8 * sector, 7 * sector, 0}, } for _, tt := range tests { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 15b199a4..04fdfa99 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -707,6 +707,45 @@ scenarios: environments: go: internal/journal/journal_test.go::TestBatchCommitReturnsSyncError + - id: checkpoint-keeps-incomplete-intents + name: Checkpoints never pass an intent that has not completed + description: A transaction committed but not yet applied stays in the journal through a checkpoint and an unmount, and the next mount replays it; completed intents are checkpointed away + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestCheckpointKeepsIncompleteIntents + + - id: pending-intent-survives-full-ring + name: A full journal never overwrites a pending intent + description: While one intent pins the head, other operations fill the ring until Commit returns ErrJournalFull; the ring never ends at the head, where it would read as empty, and a crash still replays the pending intent + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestPendingIntentSurvivesFullRing + + - id: recovery-ends-with-tail-at-end + name: Recovery ends when the tail is at the end of the data area + description: A crash with the journal tail exactly at the end of the data area replays the pending transaction and returns, instead of scanning forever + priority: go_only + environments: + go: internal/journal/checkpoint_test.go::TestRecoveryEndsWithTailAtEnd + + - id: create-replays-after-checkpoint-crash + name: A create cut off after a checkpoint is replayed + description: A create stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount creates the file and fsck is clean + priority: high + environments: + go: internal/goclusterfs/journal_crash_test.go::TestCreateReplaysAfterCheckpointAndCrash + docker: TODO # needs a pause between commit and completion, which only the Go test hook gives + vm: TODO # a power failure cannot be timed into the millisecond window between commit and completion without that hook + + - id: truncate-replay-keeps-mapped-blocks + name: Replay of an interrupted truncate keeps blocks files still map + description: A truncate stops between its journal commit and its changes, a checkpoint runs, and the process dies; replay at the next mount frees no block a file maps, and fsck is clean + priority: high + environments: + go: internal/goclusterfs/journal_crash_test.go::TestTruncateReplayKeepsMappedBlocks + docker: TODO # needs a pause between commit and completion, which only the Go test hook gives + vm: TODO # a power failure cannot be timed into the millisecond window between commit and completion without that hook + - id: intent-completion-serialize name: IntentCompletion serialization description: IntentCompletion record serializes and deserializes correctly