diff --git a/CHANGELOG.md b/CHANGELOG.md index 194876bc..a473e1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,7 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - A file node that outlived its file, such as an O_PATH descriptor or a name another node had just removed, could open, read, write and truncate the next file to take its inode slot. It now fails with ESTALE. - Snapshot lookups and listings skipped a snapshot registry sector they could not read: `getfattr -n user.snapshots` and `GET /api/v1/snapshots` returned part of the list, and a delete, clone or lookup of a snapshot in that sector reported it missing. A snapshot delete job took such a snapshot for already deleted. They now fail with `EIO`, and the API answers 500 with the sector and what to run. A snapshot in a readable sector is still found, and mount still succeeds and logs a warning naming the sector. -- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it, except when a peer unlinks a file in the moment right after another node creates it; that node can then free the file while its creator still has it open. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s. +- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s. - Opening or stat-ing a file that another caller had just unlinked failed with EISDIR, or showed the freed inode's empty attributes. It now fails with ESTALE, and the kernel looks the name up again. - An open with O_CREAT and no O_EXCL failed with EEXIST when another node or thread created the same name between the kernel's lookup and the create. It now opens that file, and truncates it with O_TRUNC. - Reads no longer show bytes nobody wrote to the file. A partial write into a newly allocated block left the rest of that block as the disk held it, often a deleted file's data, readable as soon as the file covered it; such writes now zero the block first, which doubles the bytes written for new data until a later change zeroes only what can become readable. A truncate into the middle of a block left the old bytes after the cut, and growing the file later showed them; truncate now zeroes them before it shrinks the file. diff --git a/internal/goclusterfs/eio_sweep_test.go b/internal/goclusterfs/eio_sweep_test.go index 245924a1..6fc6ffbe 100644 --- a/internal/goclusterfs/eio_sweep_test.go +++ b/internal/goclusterfs/eio_sweep_test.go @@ -267,6 +267,9 @@ func runEIO(t *testing.T, n int64, crash bool) []eioWrite { fsys.snapshotWkr.Stop() fsys.dev.SetWriteGate(func(int64) error { return errPowerCut }) fsys = mountAfterPowerCut(t, path, offset, size, false) + } else { + // A write the workload never reached must not fail fsck's repair instead. + fsys.dev.SetWriteGate(func(int64) error { return nil }) } checkEIOResults(t, fsys, ok) expectOnlyLeaks(t, fsys) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index d06299fc..54e0885c 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -1061,6 +1061,8 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin logging.Debugf("Create %q: ClaimSlot %d failed: %v", name, inodeNum, err) return nil, claimErrno(err) } + // Counted before the name appears: a peer may unlink it at once, and this node must hold the file for its caller. + f.addOpen(childIno.InodeNum, 1) now := time.Now().UnixNano() childIno.Mode = inode.S_IFREG | uint16(mode&0777) childIno.UID = uid @@ -1071,6 +1073,7 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin childIno.Ctime = now if err := f.inodeTbl.Write(ctx, childIno); err != nil { logging.Debugf("Create %q: Write inode %d failed: %v", name, childIno.InodeNum, err) + f.addOpen(childIno.InodeNum, -1) f.freeNewInode(ctx, childIno.InodeNum) return nil, syscall.EIO } @@ -1080,13 +1083,16 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin if err := f.dirMgr.Add(ctx, dirIno, name, uint64(inodeNum), dir.DT_REG); err != nil { logging.Debugf("Create %q: dirMgr.Add failed: %v", name, err) + f.addOpen(childIno.InodeNum, -1) f.freeNewInode(ctx, childIno.InodeNum) return nil, nameErrno(err) } + if afterCommit != nil { + afterCommit("create: named") + } f.cacheInode(childIno) f.touchParent(ctx, parentNum, 0, now) - f.addOpen(childIno.InodeNum, 1) ch := parent.NewPersistentInode(ctx, newFile(f, childIno), fs.StableAttr{ Mode: fuse.S_IFREG, Ino: uint64(childIno.InodeNum), diff --git a/internal/goclusterfs/orphans_test.go b/internal/goclusterfs/orphans_test.go index b6796428..70d330ad 100644 --- a/internal/goclusterfs/orphans_test.go +++ b/internal/goclusterfs/orphans_test.go @@ -343,3 +343,31 @@ func TestOwnPassKeepsAReservedOrphan(t *testing.T) { t.Errorf("ForgetSlot of this node's slot kept %v, a still-linked entry no unlink is working on", got) } } + +// A peer may unlink a new file as soon as its name appears; the creating node must still hold the file for its caller. +func TestCreateHoldsAFileAPeerUnlinksAtOnce(t *testing.T) { + path := newFormattedImage(t) + cluster := newFakeCluster(0, 1) + nodeA := cluster.join(t, withBridge(openFormattedFS(t, path)), 0) + nodeB := cluster.join(t, openFormattedFS(t, path), 1) + ctx := context.Background() + var once sync.Once + afterCommit = func(op string) { + if op == "create: named" { + once.Do(func() { + if errno := nodeB.Unlink(ctx, "f"); errno != 0 { + t.Errorf("node B unlink: %v", errno) + } + nodeA.LearnOrphans(1, cluster.list(1)) + }) + } + } + t.Cleanup(func() { afterCommit = nil }) + + f := createOpen(t, nodeA, "f") + writeAt(t, f, pattern(1, 4096), 0) + checkAt(t, f, pattern(1, 4096), 0) + if ino := readInode(t, nodeA, f.inodeNum); ino.Magic != inode.InodeMagic || ino.LinkCount != 0 || ino.Holders != 1 { + t.Fatalf("magic %#x, links %d, holders %#b; want an orphan node A holds", ino.Magic, ino.LinkCount, ino.Holders) + } +} diff --git a/tests/integration/coherence_history_test.go b/tests/integration/coherence_history_test.go index 53c1b88c..d08332ca 100644 --- a/tests/integration/coherence_history_test.go +++ b/tests/integration/coherence_history_test.go @@ -46,7 +46,7 @@ func TestCoherenceHistory(t *testing.T) { } out := os.Getenv("GCFS_HISTORY_OUT") - cluster := NewMultiNodeCluster(t, nodes, 256) + cluster := NewMultiNodeCluster(t, nodes, 1024) if err := cluster.FormatAndStartFirst(false); err != nil { t.Fatalf("format and start node 0: %v", err) }