From cd5dc333dfaf8b9702f0895735e0a12b7d283fa1 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 14:10:23 -0600 Subject: [PATCH 1/4] fix(fs): count a new file's open before its name appears createFileOnce counted the caller's open only after adding the name and touching the parent. A peer that unlinked the name in that window left an orphan this node did not yet count as open, so its next orphan pass freed the file, and the caller's first write went to a freed or reused inode. The open is now counted right after the slot is claimed. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/fs.go | 8 +++++++- internal/goclusterfs/orphans_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index d06299fc..54e0885c 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -1061,6 +1061,8 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin logging.Debugf("Create %q: ClaimSlot %d failed: %v", name, inodeNum, err) return nil, claimErrno(err) } + // Counted before the name appears: a peer may unlink it at once, and this node must hold the file for its caller. + f.addOpen(childIno.InodeNum, 1) now := time.Now().UnixNano() childIno.Mode = inode.S_IFREG | uint16(mode&0777) childIno.UID = uid @@ -1071,6 +1073,7 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin childIno.Ctime = now if err := f.inodeTbl.Write(ctx, childIno); err != nil { logging.Debugf("Create %q: Write inode %d failed: %v", name, childIno.InodeNum, err) + f.addOpen(childIno.InodeNum, -1) f.freeNewInode(ctx, childIno.InodeNum) return nil, syscall.EIO } @@ -1080,13 +1083,16 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin if err := f.dirMgr.Add(ctx, dirIno, name, uint64(inodeNum), dir.DT_REG); err != nil { logging.Debugf("Create %q: dirMgr.Add failed: %v", name, err) + f.addOpen(childIno.InodeNum, -1) f.freeNewInode(ctx, childIno.InodeNum) return nil, nameErrno(err) } + if afterCommit != nil { + afterCommit("create: named") + } f.cacheInode(childIno) f.touchParent(ctx, parentNum, 0, now) - f.addOpen(childIno.InodeNum, 1) ch := parent.NewPersistentInode(ctx, newFile(f, childIno), fs.StableAttr{ Mode: fuse.S_IFREG, Ino: uint64(childIno.InodeNum), diff --git a/internal/goclusterfs/orphans_test.go b/internal/goclusterfs/orphans_test.go index b6796428..70d330ad 100644 --- a/internal/goclusterfs/orphans_test.go +++ b/internal/goclusterfs/orphans_test.go @@ -343,3 +343,31 @@ func TestOwnPassKeepsAReservedOrphan(t *testing.T) { t.Errorf("ForgetSlot of this node's slot kept %v, a still-linked entry no unlink is working on", got) } } + +// A peer may unlink a new file as soon as its name appears; the creating node must still hold the file for its caller. +func TestCreateHoldsAFileAPeerUnlinksAtOnce(t *testing.T) { + path := newFormattedImage(t) + cluster := newFakeCluster(0, 1) + nodeA := cluster.join(t, withBridge(openFormattedFS(t, path)), 0) + nodeB := cluster.join(t, openFormattedFS(t, path), 1) + ctx := context.Background() + var once sync.Once + afterCommit = func(op string) { + if op == "create: named" { + once.Do(func() { + if errno := nodeB.Unlink(ctx, "f"); errno != 0 { + t.Errorf("node B unlink: %v", errno) + } + nodeA.LearnOrphans(1, cluster.list(1)) + }) + } + } + t.Cleanup(func() { afterCommit = nil }) + + f := createOpen(t, nodeA, "f") + writeAt(t, f, pattern(1, 4096), 0) + checkAt(t, f, pattern(1, 4096), 0) + if ino := readInode(t, nodeA, f.inodeNum); ino.Magic != inode.InodeMagic || ino.LinkCount != 0 || ino.Holders != 1 { + t.Fatalf("magic %#x, links %d, holders %#b; want an orphan node A holds", ino.Magic, ino.LinkCount, ino.Holders) + } +} From d90c61b6fe22148520ae72839a4b72ea81416f25 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 14:19:01 -0600 Subject: [PATCH 2/4] test(history): give the checker a 1 GB volume At 256 MB the volume has 257 one-MB blocks, so writes failed with ENOSPC under churn while peers had not yet let go of removed files; fsck showed no leak. Co-Authored-By: Claude Opus 5.5 --- tests/integration/coherence_history_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/coherence_history_test.go b/tests/integration/coherence_history_test.go index 53c1b88c..d08332ca 100644 --- a/tests/integration/coherence_history_test.go +++ b/tests/integration/coherence_history_test.go @@ -46,7 +46,7 @@ func TestCoherenceHistory(t *testing.T) { } out := os.Getenv("GCFS_HISTORY_OUT") - cluster := NewMultiNodeCluster(t, nodes, 256) + cluster := NewMultiNodeCluster(t, nodes, 1024) if err := cluster.FormatAndStartFirst(false); err != nil { t.Fatalf("format and start node 0: %v", err) } From 0fe7a4d3318e3d3f0cabdc52e055b546046d7342 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 15:19:26 -0600 Subject: [PATCH 3/4] docs(changelog): unlink-while-open now covers a file a peer unlinks right after its creation Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 194876bc..a473e1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,7 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - A file node that outlived its file, such as an O_PATH descriptor or a name another node had just removed, could open, read, write and truncate the next file to take its inode slot. It now fails with ESTALE. - Snapshot lookups and listings skipped a snapshot registry sector they could not read: `getfattr -n user.snapshots` and `GET /api/v1/snapshots` returned part of the list, and a delete, clone or lookup of a snapshot in that sector reported it missing. A snapshot delete job took such a snapshot for already deleted. They now fail with `EIO`, and the API answers 500 with the sector and what to run. A snapshot in a readable sector is still found, and mount still succeeds and logs a warning naming the sector. -- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it, except when a peer unlinks a file in the moment right after another node creates it; that node can then free the file while its creator still has it open. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s. +- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s. - Opening or stat-ing a file that another caller had just unlinked failed with EISDIR, or showed the freed inode's empty attributes. It now fails with ESTALE, and the kernel looks the name up again. - An open with O_CREAT and no O_EXCL failed with EEXIST when another node or thread created the same name between the kernel's lookup and the create. It now opens that file, and truncates it with O_TRUNC. - Reads no longer show bytes nobody wrote to the file. A partial write into a newly allocated block left the rest of that block as the disk held it, often a deleted file's data, readable as soon as the file covered it; such writes now zero the block first, which doubles the bytes written for new data until a later change zeroes only what can become readable. A truncate into the middle of a block left the old bytes after the cut, and growing the file later showed them; truncate now zeroes them before it shrinks the file. From 549301433da4a435eb51ab3ac135072a9f8a898e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 15:44:31 -0600 Subject: [PATCH 4/4] test(fs): keep the EIO sweep's injected failure out of fsck repair When the race detector's timing left the workload short of the chosen write, the EIO landed in fsck's repair, which then left orphans and blocks it should have freed. Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/eio_sweep_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/goclusterfs/eio_sweep_test.go b/internal/goclusterfs/eio_sweep_test.go index 245924a1..6fc6ffbe 100644 --- a/internal/goclusterfs/eio_sweep_test.go +++ b/internal/goclusterfs/eio_sweep_test.go @@ -267,6 +267,9 @@ func runEIO(t *testing.T, n int64, crash bool) []eioWrite { fsys.snapshotWkr.Stop() fsys.dev.SetWriteGate(func(int64) error { return errPowerCut }) fsys = mountAfterPowerCut(t, path, offset, size, false) + } else { + // A write the workload never reached must not fail fsck's repair instead. + fsys.dev.SetWriteGate(func(int64) error { return nil }) } checkEIOResults(t, fsys, ok) expectOnlyLeaks(t, fsys)