Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions internal/goclusterfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,16 @@ func (f *FS) Create(ctx context.Context, name string, flags uint32, mode uint32,
return nil, nil, 0, syscall.EAGAIN
}

// closeIntent completes an intent its operation left open by returning early, so a failed operation is never replayed.
func (f *FS) closeIntent(seq uint64) {
if err := f.journal.CompleteIntent(seq); err != nil {
log.Printf("[WARN] journal: completing intent %d failed: %v", seq, err)
}
}

// afterCommit runs between an operation's journal commit and its first change; tests stop there as a crash would.
var afterCommit func(op string)

// createFileInternal implements file creation for the root directory.
func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (*fs.Inode, fs.FileHandle, uint32, syscall.Errno) {
inodeNum, err := f.inodeTbl.FindFreeSlot()
Expand Down Expand Up @@ -721,7 +731,7 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32,
txn = f.journal.Begin()
if err := txn.LogInodeCreate(&journal.InodeCreatePayload{
InodeNum: uint64(inodeNum),
Mode: mode,
Mode: uint32(inode.S_IFREG) | (mode & 0777),
UID: uid,
GID: gid,
ParentInodeNum: uint64(inode.InodeRoot),
Expand All @@ -745,6 +755,10 @@ func (f *FS) createFileInternal(ctx context.Context, name string, flags uint32,
if commitErr != nil {
return nil, nil, 0, syscall.EIO
}
defer f.closeIntent(intentSeq)
if afterCommit != nil {
afterCommit("create")
}
}
// === End WAL transaction - intent is now durable ===

Expand Down Expand Up @@ -874,7 +888,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f
txn = f.journal.Begin()
if err := txn.LogInodeCreate(&journal.InodeCreatePayload{
InodeNum: uint64(inodeNum),
Mode: mode | uint32(inode.S_IFDIR),
Mode: uint32(inode.S_IFDIR) | (mode & 0777),
UID: uid,
GID: gid,
ParentInodeNum: uint64(inode.InodeRoot),
Expand All @@ -898,6 +912,7 @@ func (f *FS) mkdirInternal(ctx context.Context, name string, mode uint32, out *f
if commitErr != nil {
return nil, syscall.EIO
}
defer f.closeIntent(intentSeq)
}
// === End WAL transaction - intent is now durable ===

Expand Down Expand Up @@ -1033,6 +1048,7 @@ func (f *FS) Unlink(ctx context.Context, name string) syscall.Errno {
if commitErr != nil {
return syscall.EIO
}
defer f.closeIntent(intentSeq)
}
// === End WAL transaction - intent is now durable ===

Expand Down Expand Up @@ -1126,6 +1142,7 @@ func (f *FS) Rmdir(ctx context.Context, name string) syscall.Errno {
if commitErr != nil {
return syscall.EIO
}
defer f.closeIntent(intentSeq)
}
// === End WAL transaction - intent is now durable ===

Expand Down Expand Up @@ -1468,7 +1485,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod
// Log inode creation intent
if err := txn.LogInodeCreate(&journal.InodeCreatePayload{
InodeNum: uint64(inodeNum),
Mode: mode,
Mode: uint32(inode.S_IFREG) | (mode & 0777),
UID: uid,
GID: gid,
ParentInodeNum: uint64(d.inodeNum),
Expand Down Expand Up @@ -1498,6 +1515,7 @@ func (d *Dir) createInternal(ctx context.Context, name string, flags uint32, mod
log.Printf("[DEBUG] Create %q: journal commit failed: %v", name, commitErr)
return nil, nil, 0, syscall.EIO
}
defer d.fs.closeIntent(intentSeq)
}

// Phase 3: Add directory entry FIRST (may acquire BlockBitmap lock for directory expansion)
Expand Down Expand Up @@ -1709,6 +1727,7 @@ func (d *Dir) mkdirInternal(ctx context.Context, name string, mode uint32, out *
log.Printf("[DEBUG] Mkdir %q: journal commit failed: %v", name, commitErr)
return nil, syscall.EIO
}
defer d.fs.closeIntent(intentSeq)
}

// Phase 3: Add directory entry in parent FIRST (may acquire BlockBitmap lock)
Expand Down Expand Up @@ -1965,6 +1984,7 @@ func (d *Dir) removeEntry(ctx context.Context, name string, isDir bool) syscall.
if commitErr != nil {
return syscall.EIO
}
defer d.fs.closeIntent(intentSeq)
}
// === End WAL transaction ===

Expand Down Expand Up @@ -2844,6 +2864,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i
log.Printf("[DEBUG] Write inode %d: journal commit failed: %v", f.inodeNum, commitErr)
return 0, syscall.EIO
}
defer f.fs.closeIntent(intentSeq)
txn = nil // Mark as committed so defer doesn't abort
}
// === End WAL transaction ===
Expand Down Expand Up @@ -3066,6 +3087,7 @@ func (f *File) migrateToExtent(ctx context.Context, ino *inode.Inode) error {
_ = f.fs.bitmapMgr.Free(ctx, newBlock)
return commitErr
}
defer f.fs.closeIntent(intentSeq)
}

ino.InlineExtents[0] = inode.InlineExtent{PhysicalBlock: newBlock, Length: 1, Flags: extent.FlagAllocated}
Expand Down Expand Up @@ -3214,6 +3236,10 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn
if commitErr != nil {
return syscall.EIO, nil
}
defer f.fs.closeIntent(intentSeq)
if afterCommit != nil {
afterCommit("setattr")
}
}
// === End WAL transaction ===

Expand Down
99 changes: 99 additions & 0 deletions internal/goclusterfs/journal_crash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package goclusterfs

import (
"context"
"testing"

"github.com/hanwen/go-fuse/v2/fuse"

"github.com/supporttools/GoClusterFS/internal/fsck"
"github.com/supporttools/GoClusterFS/internal/journal"
"github.com/supporttools/GoClusterFS/internal/superblock"
)

// journaledFS opens a formatted image with a root directory and a journal in slot 0's region.
func journaledFS(t *testing.T) (fsys *FS, path string, offset, size uint64) {
t.Helper()
path = newFormattedImage(t)
fsys = openFormattedFS(t, path)
if err := fsys.InitRoot(context.Background()); err != nil {
t.Fatal(err)
}
sb, err := superblock.Read(fsys.dev)
if err != nil {
t.Fatal(err)
}
offset, size = sb.JournalRegionOffset, sb.JournalSlotSize()
if fsys.journal, err = journal.Init(fsys.dev, offset, size, fsys.nodeUUID, 1, journal.DefaultConfig()); err != nil {
t.Fatal(err)
}
return fsys, path, offset, size
}

// crashAfterCommit makes the next op stop after its commit, as a crash would, once a checkpoint has run.
func crashAfterCommit(t *testing.T, fsys *FS, op string) <-chan struct{} {
t.Helper()
crashed := make(chan struct{})
afterCommit = func(got string) {
if got != op {
return
}
if _, err := fsys.journal.Checkpoint(); err != nil {
t.Errorf("checkpoint: %v", err)
}
close(crashed)
select {}
}
t.Cleanup(func() { afterCommit = nil })
return crashed
}

// remountAndReplay opens the image as the next mount would and replays the journal.
func remountAndReplay(t *testing.T, path string, offset, size uint64) *FS {
t.Helper()
fsys := openFormattedFS(t, path)
replay := journal.CreateTxnRecoveryHandler(&journal.RecoveryContext{
Dev: fsys.dev, InodeTbl: fsys.inodeTbl, BitmapMgr: fsys.bitmapMgr,
DirMgr: fsys.dirMgr, ExtentMgr: fsys.extentMgr,
})
if _, err := journal.ScanAndRecover(fsys.dev, offset, size, replay); err != nil {
t.Fatalf("replay: %v", err)
}
return fsys
}

func expectCleanFsck(t *testing.T, fsys *FS) {
t.Helper()
if r := runFsck(t, fsys, fsck.DefaultOptions()); len(r.Problems) > 0 {
t.Fatalf("fsck after replay:%s", problemList(r))
}
}

// A checkpoint between a create's commit and its changes, then a crash: the next mount must still replay the create.
func TestCreateReplaysAfterCheckpointAndCrash(t *testing.T) {
fsys, path, offset, size := journaledFS(t)
crashed := crashAfterCommit(t, fsys, "create")
go fsys.Create(context.Background(), "crashed.txt", 0, 0o644, &fuse.EntryOut{})
<-crashed

remounted := remountAndReplay(t, path, offset, size)
lookupFile(t, remounted, "crashed.txt")
expectCleanFsck(t, remounted)
}

// A crash after a truncate's commit: replay must not free blocks that files still map.
func TestTruncateReplayKeepsMappedBlocks(t *testing.T) {
fsys, path, offset, size := journaledFS(t)
victim := createFile(t, fsys, "victim.bin")
writeAt(t, victim, pattern(1, 3*BlockSize), 0)
other := createFile(t, fsys, "other.bin")
writeAt(t, other, pattern(2, 2*BlockSize), 0)

crashed := crashAfterCommit(t, fsys, "setattr")
go tryTruncate(victim, BlockSize)
<-crashed

remounted := remountAndReplay(t, path, offset, size)
expectCleanFsck(t, remounted)
checkFile(t, lookupFile(t, remounted, "other.bin"), pattern(2, 2*BlockSize))
}
160 changes: 160 additions & 0 deletions internal/journal/checkpoint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package journal

import (
"errors"
"slices"
"testing"
"time"
)

// commitIntents commits one transaction per entry and completes those marked true.
func commitIntents(t *testing.T, j *Journal, complete ...bool) []uint64 {
t.Helper()
var seqs []uint64
for i, done := range complete {
txn := j.Begin()
if err := txn.LogInodeCreate(testInodePayload(uint64(100 + i))); err != nil {
t.Fatal(err)
}
seq, err := j.Commit(txn)
if err != nil {
t.Fatal(err)
}
if done {
if err := j.CompleteIntent(seq); err != nil {
t.Fatal(err)
}
}
seqs = append(seqs, seq)
}
return seqs
}

// replayedAfterCrash reopens the region as the next mount would and returns the transactions it replays.
func replayedAfterCrash(t *testing.T, j *Journal) []uint64 {
t.Helper()
region, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize)
if err != nil {
t.Fatal(err)
}
var replayed []uint64
if _, err := Recover(region, func(txn *RecoveredTxn) error {
replayed = append(replayed, txn.Header.Seq)
return nil
}); err != nil {
t.Fatal(err)
}
return replayed
}

func newTestJournal(t *testing.T) *Journal {
t.Helper()
dev, cleanup := createTestDevice(t, 10*1024*1024)
t.Cleanup(cleanup)
j, err := Init(dev, 0, DefaultJournalSlotSize, [16]byte{1}, 1, DefaultConfig())
if err != nil {
t.Fatal(err)
}
return j
}

func TestCheckpointKeepsIncompleteIntents(t *testing.T) {
j := newTestJournal(t)
seqs := commitIntents(t, j, true, false, true)
if _, err := j.Checkpoint(); err != nil {
t.Fatal(err)
}
if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs[1:2]) {
t.Errorf("replayed %v after a checkpoint and a crash, want the incomplete intent %d", got, seqs[1])
}
}

func TestCheckpointPassesCompletedIntents(t *testing.T) {
j := newTestJournal(t)
seqs := commitIntents(t, j, true, false)
if err := j.CompleteIntent(seqs[1]); err != nil {
t.Fatal(err)
}
if done, err := j.Checkpoint(); err != nil || !done {
t.Fatalf("Checkpoint = %v, %v; want the head moved", done, err)
}
if got := replayedAfterCrash(t, j); len(got) > 0 {
t.Errorf("replayed %v after checkpointing completed intents", got)
}
}

func TestStopLeavesIncompleteIntentsForReplay(t *testing.T) {
j := newTestJournal(t)
j.Start()
seqs := commitIntents(t, j, false)
if err := j.Stop(); err != nil {
t.Fatal(err)
}
if got := replayedAfterCrash(t, j); !slices.Equal(got, seqs) {
t.Errorf("replayed %v after unmounting with an incomplete intent, want %v", got, seqs)
}
}

// With the head pinned by a pending intent, other operations fill the ring until the journal is full, never past the intent.
func TestPendingIntentSurvivesFullRing(t *testing.T) {
j := newTestJournal(t)
commitIntents(t, j, slices.Repeat([]bool{true}, 100)...)
if _, err := j.Checkpoint(); err != nil {
t.Fatal(err)
}
pinned := commitIntents(t, j, false)[0]
for n := 1; ; n++ {
txn := j.Begin()
if err := txn.LogInodeCreate(testInodePayload(uint64(n))); err != nil {
t.Fatal(err)
}
seq, err := j.Commit(txn)
if errors.Is(err, ErrJournalFull) {
break
}
if err != nil {
t.Fatalf("op %d: %v", n, err)
}
if err := j.CompleteIntent(seq); err != nil && !errors.Is(err, ErrJournalFull) {
t.Fatalf("op %d: %v", n, err)
}
if _, err := j.Checkpoint(); err != nil {
t.Fatal(err)
}
if head, tail := j.region.HeadOffset(), j.region.TailOffset(); head == tail {
t.Fatalf("op %d: head == tail == %d with intent %d pending, which reads as an empty journal", n, head, pinned)
}
}
if got := replayedAfterCrash(t, j); !slices.Contains(got, pinned) {
t.Errorf("a full journal replayed %v, want the pending intent %d", got, pinned)
}
}

// A crash with the tail at the very end of the data area must not hang the next mount's scan.
func TestRecoveryEndsWithTailAtEnd(t *testing.T) {
dev, cleanup := createTestDevice(t, 1024*1024)
t.Cleanup(cleanup)
j, err := Init(dev, 0, uint64(JournalDataOffset)+3*4096, [16]byte{1}, 1, DefaultConfig())
if err != nil {
t.Fatal(err)
}
commitIntents(t, j, true)
if _, err := j.Checkpoint(); err != nil {
t.Fatal(err)
}
pending := commitIntents(t, j, false)
if tail := j.region.TailOffset(); tail != j.region.DataAreaSize() {
t.Fatalf("tail %d, want the end of the %d-byte data area", tail, j.region.DataAreaSize())
}

done := make(chan []uint64, 1)
go func() { done <- replayedAfterCrash(t, j) }()
select {
case got := <-done:
if !slices.Equal(got, pending) {
t.Errorf("replayed %v, want %v", got, pending)
}
case <-time.After(10 * time.Second):
t.Fatal("recovery scan still running after 10s with the tail at the end of the data area")
}
}
Loading
Loading