Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

- A file node that outlived its file, such as an O_PATH descriptor or a name another node had just removed, could open, read, write and truncate the next file to take its inode slot. It now fails with ESTALE.
- Snapshot lookups and listings skipped a snapshot registry sector they could not read: `getfattr -n user.snapshots` and `GET /api/v1/snapshots` returned part of the list, and a delete, clone or lookup of a snapshot in that sector reported it missing. A snapshot delete job took such a snapshot for already deleted. They now fail with `EIO`, and the API answers 500 with the sector and what to run. A snapshot in a readable sector is still found, and mount still succeeds and logs a warning naming the sector.
- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it, except when a peer unlinks a file in the moment right after another node creates it; that node can then free the file while its creator still has it open. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s.
- A file unlinked while open, on this node or another, was freed at once: its descriptors read and wrote whatever file took the inode next. The file now keeps its inode and blocks until every node has closed it. A node that dies holding it lets go when a peer replays its journal. While other nodes are mounted, each node tracks up to 487 unlinked files that other nodes have not yet let go of. An unlink beyond that waits, and fails with EBUSY only when live nodes keep holding them for 20 s.
- Opening or stat-ing a file that another caller had just unlinked failed with EISDIR, or showed the freed inode's empty attributes. It now fails with ESTALE, and the kernel looks the name up again.
- An open with O_CREAT and no O_EXCL failed with EEXIST when another node or thread created the same name between the kernel's lookup and the create. It now opens that file, and truncates it with O_TRUNC.
- Reads no longer show bytes nobody wrote to the file. A partial write into a newly allocated block left the rest of that block as the disk held it, often a deleted file's data, readable as soon as the file covered it; such writes now zero the block first, which doubles the bytes written for new data until a later change zeroes only what can become readable. A truncate into the middle of a block left the old bytes after the cut, and growing the file later showed them; truncate now zeroes them before it shrinks the file.
Expand Down
3 changes: 3 additions & 0 deletions internal/goclusterfs/eio_sweep_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,9 @@ func runEIO(t *testing.T, n int64, crash bool) []eioWrite {
fsys.snapshotWkr.Stop()
fsys.dev.SetWriteGate(func(int64) error { return errPowerCut })
fsys = mountAfterPowerCut(t, path, offset, size, false)
} else {
// A write the workload never reached must not fail fsck's repair instead.
fsys.dev.SetWriteGate(func(int64) error { return nil })
}
checkEIOResults(t, fsys, ok)
expectOnlyLeaks(t, fsys)
Expand Down
8 changes: 7 additions & 1 deletion internal/goclusterfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -1061,6 +1061,8 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin
logging.Debugf("Create %q: ClaimSlot %d failed: %v", name, inodeNum, err)
return nil, claimErrno(err)
}
// Counted before the name appears: a peer may unlink it at once, and this node must hold the file for its caller.
f.addOpen(childIno.InodeNum, 1)
now := time.Now().UnixNano()
childIno.Mode = inode.S_IFREG | uint16(mode&0777)
childIno.UID = uid
Expand All @@ -1071,6 +1073,7 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin
childIno.Ctime = now
if err := f.inodeTbl.Write(ctx, childIno); err != nil {
logging.Debugf("Create %q: Write inode %d failed: %v", name, childIno.InodeNum, err)
f.addOpen(childIno.InodeNum, -1)
f.freeNewInode(ctx, childIno.InodeNum)
return nil, syscall.EIO
}
Expand All @@ -1080,13 +1083,16 @@ func (f *FS) createFileOnce(ctx context.Context, parent *fs.Inode, parentNum uin

if err := f.dirMgr.Add(ctx, dirIno, name, uint64(inodeNum), dir.DT_REG); err != nil {
logging.Debugf("Create %q: dirMgr.Add failed: %v", name, err)
f.addOpen(childIno.InodeNum, -1)
f.freeNewInode(ctx, childIno.InodeNum)
return nil, nameErrno(err)
}
if afterCommit != nil {
afterCommit("create: named")
}
f.cacheInode(childIno)
f.touchParent(ctx, parentNum, 0, now)

f.addOpen(childIno.InodeNum, 1)
ch := parent.NewPersistentInode(ctx, newFile(f, childIno), fs.StableAttr{
Mode: fuse.S_IFREG,
Ino: uint64(childIno.InodeNum),
Expand Down
28 changes: 28 additions & 0 deletions internal/goclusterfs/orphans_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,3 +343,31 @@ func TestOwnPassKeepsAReservedOrphan(t *testing.T) {
t.Errorf("ForgetSlot of this node's slot kept %v, a still-linked entry no unlink is working on", got)
}
}

// A peer may unlink a new file as soon as its name appears; the creating node must still hold the file for its caller.
func TestCreateHoldsAFileAPeerUnlinksAtOnce(t *testing.T) {
path := newFormattedImage(t)
cluster := newFakeCluster(0, 1)
nodeA := cluster.join(t, withBridge(openFormattedFS(t, path)), 0)
nodeB := cluster.join(t, openFormattedFS(t, path), 1)
ctx := context.Background()
var once sync.Once
afterCommit = func(op string) {
if op == "create: named" {
once.Do(func() {
if errno := nodeB.Unlink(ctx, "f"); errno != 0 {
t.Errorf("node B unlink: %v", errno)
}
nodeA.LearnOrphans(1, cluster.list(1))
})
}
}
t.Cleanup(func() { afterCommit = nil })

f := createOpen(t, nodeA, "f")
writeAt(t, f, pattern(1, 4096), 0)
checkAt(t, f, pattern(1, 4096), 0)
if ino := readInode(t, nodeA, f.inodeNum); ino.Magic != inode.InodeMagic || ino.LinkCount != 0 || ino.Holders != 1 {
t.Fatalf("magic %#x, links %d, holders %#b; want an orphan node A holds", ino.Magic, ino.LinkCount, ino.Holders)
}
}
2 changes: 1 addition & 1 deletion tests/integration/coherence_history_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func TestCoherenceHistory(t *testing.T) {
}
out := os.Getenv("GCFS_HISTORY_OUT")

cluster := NewMultiNodeCluster(t, nodes, 256)
cluster := NewMultiNodeCluster(t, nodes, 1024)
if err := cluster.FormatAndStartFirst(false); err != nil {
t.Fatalf("format and start node 0: %v", err)
}
Expand Down
Loading