Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Fixed

- `chmod`, `chown` and `touch` could undo another node's concurrent change to the same inode: on a directory, a create's link count or mtime; on a file, the size an append inside an allocated block set. Directory attribute changes now hold the inode lock from read to write, and file attribute changes retry when the inode changed since they read it.
- `chmod` cleared a file's type bits, so the file could no longer be opened ("Is a directory") and fsck reported an invalid file type. `chmod`, `chown` and `touch` now also work on directories and the mount root, which returned "Operation not supported".
- The cache summary logged at unmount showed hit rates 100 times too high.
- All nodes wrote the same journal region. A restarting node replayed and then reset the journal of nodes still writing, and no node could replay a crashed node's journal. Each node registry slot now owns a 4 MB journal, and a crashed node's journal is replayed through its slot.
Expand Down
15 changes: 13 additions & 2 deletions internal/goclusterfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -760,6 +760,9 @@ func (f *FS) dropNewInode(ctx context.Context, ino *inode.Inode) {
// testHookBeforeMkdirInodeWrite, when set by a test, fails a mkdir's inode write with its error.
var testHookBeforeMkdirInodeWrite func() error

// testHookAfterSetattrRead, when set by a test, runs between a setattr's read of the inode and its write.
var testHookAfterSetattrRead func()

func injectedMkdirFailure() error {
if testHookBeforeMkdirInodeWrite != nil {
return testHookBeforeMkdirInodeWrite()
Expand Down Expand Up @@ -1112,7 +1115,10 @@ func (f *FS) setattrDir(ctx context.Context, num uint32, in *fuse.SetAttrIn) sys
return syscall.EISDIR
}
now := time.Now().UnixNano()
updated, err := f.updateInode(ctx, num, func(ino *inode.Inode) error {
updated, err := f.updateInodeLocked(ctx, num, func(ino *inode.Inode) error {
if testHookAfterSetattrRead != nil {
testHookAfterSetattrRead()
}
if mode, ok := in.GetMode(); ok {
ino.Mode = inode.S_IFDIR | uint16(mode&0777)
ino.Ctime = now
Expand Down Expand Up @@ -2659,6 +2665,9 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn
if err != nil {
return syscall.EIO, nil
}
if testHookAfterSetattrRead != nil {
testHookAfterSetattrRead()
}

if _, resizing := in.GetSize(); resizing && ino.IsSFBBacked() {
if err := f.migrateToExtent(ctx, ino); err != nil {
Expand All @@ -2672,6 +2681,7 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn
return syscall.EIO, nil
}
}
readGen := ino.MetaGeneration

now := time.Now().UnixNano()

Expand Down Expand Up @@ -2787,7 +2797,8 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn
ino.Mtime = mtime.UnixNano()
}

if err := f.fs.inodeTbl.Write(ctx, ino); err != nil {
// Another node's write, such as an append inside an allocated block, may have changed the size since ino was read.
if err := f.fs.inodeTbl.WriteIfUnchanged(ctx, ino, readGen); err != nil {
f.fs.extentMgr.DiscardPending(ctx, ino)
if errors.Is(err, extent.ErrTreeConflict) {
// Nothing was applied, so close the intent before the retry logs a new one.
Expand Down
134 changes: 134 additions & 0 deletions internal/goclusterfs/setattr_race_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package goclusterfs

import (
"bytes"
"context"
"syscall"
"testing"
"time"

"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)

// raceSetattr runs other between the next setattr's read of the inode and its
// write. It waits up to a second for other, so other finishes first unless the
// setattr holds the inode lock across its read and write.
func raceSetattr(t *testing.T, other func() syscall.Errno) <-chan syscall.Errno {
t.Helper()
done := make(chan syscall.Errno, 1)
testHookAfterSetattrRead = func() {
testHookAfterSetattrRead = nil
go func() { done <- other() }()
select {
case errno := <-done:
done <- errno
case <-time.After(time.Second):
}
}
t.Cleanup(func() { testHookAfterSetattrRead = nil })
return done
}

// withBridge gives fsys a go-fuse bridge without mounting it, so creates can make nodes.
func withBridge(fsys *FS) *FS {
fs.NewNodeFS(fsys, &fs.Options{})
return fsys
}

func chmodIn(mode uint32) *fuse.SetAttrIn {
in := &fuse.SetAttrIn{}
in.Valid = fuse.FATTR_MODE
in.Mode = mode
return in
}

// Node A changes a directory's mode while node B makes a directory in it.
// Node B's link count and mtime must survive node A's write.
func TestDirSetattrKeepsOtherNodesMkdir(t *testing.T) {
path := newFormattedImage(t)
nodeA, nodeB := withBridge(openFormattedFS(t, path)), withBridge(openFormattedFS(t, path))
ctx := context.Background()

var out fuse.EntryOut
if _, errno := nodeA.Mkdir(ctx, "d", 0o755, &out); errno != 0 {
t.Fatalf("mkdir d: %v", errno)
}
num := uint32(out.Ino)
before, err := nodeA.inodeTbl.Read(num)
if err != nil {
t.Fatal(err)
}

made := raceSetattr(t, func() syscall.Errno {
_, errno := nodeB.makeDir(ctx, &nodeB.Inode, num, "sub", 0o755, &fuse.EntryOut{})
return errno
})
if errno := (&Dir{fs: nodeA, inodeNum: num}).Setattr(ctx, nil, chmodIn(0o700), &fuse.AttrOut{}); errno != 0 {
t.Fatalf("chmod d on node A: %v", errno)
}
if errno := <-made; errno != 0 {
t.Fatalf("mkdir d/sub on node B: %v", errno)
}

after, err := nodeA.inodeTbl.Read(num)
if err != nil {
t.Fatal(err)
}
if after.Mode&0o777 != 0o700 {
t.Errorf("mode %o after chmod 700", after.Mode&0o777)
}
if after.LinkCount != before.LinkCount+1 {
t.Errorf("link count %d after node B's mkdir, want %d", after.LinkCount, before.LinkCount+1)
}
if after.Mtime <= before.Mtime {
t.Errorf("mtime %d after node B's mkdir, not later than %d", after.Mtime, before.Mtime)
}
}

// Node A changes a file's mode while node B appends inside the file's only
// block, which changes the size but not the extents. Node A must keep that size.
func TestFileSetattrKeepsOtherNodesSize(t *testing.T) {
path := newFormattedImage(t)
nodeA, nodeB := openFormattedFS(t, path), openFormattedFS(t, path)
ctx := context.Background()

fA := newRegularFile(t, nodeA)
first := pattern(1, 100)
writeAt(t, fA, first, 0)
fB := &File{fs: nodeB, inodeNum: fA.inodeNum}
appended := pattern(2, 100)

wrote := raceSetattr(t, func() syscall.Errno {
n, errno := fB.Write(ctx, nil, appended, int64(len(first)))
if errno == 0 && int(n) != len(appended) {
return syscall.EIO
}
return errno
})
if errno := fA.Setattr(ctx, nil, chmodIn(0o600), &fuse.AttrOut{}); errno != 0 {
t.Fatalf("chmod on node A: %v", errno)
}
if errno := <-wrote; errno != 0 {
t.Fatalf("append on node B: %v", errno)
}

ino, err := nodeA.inodeTbl.Read(fA.inodeNum)
if err != nil {
t.Fatal(err)
}
if ino.Mode&0o777 != 0o600 {
t.Errorf("mode %o after chmod 600", ino.Mode&0o777)
}
if want := uint64(len(first) + len(appended)); ino.Size != want {
t.Fatalf("size %d after node B's append, want %d", ino.Size, want)
}
res, errno := fA.Read(ctx, nil, make([]byte, ino.Size), 0)
if errno != 0 {
t.Fatalf("read: %v", errno)
}
got, _ := res.Bytes(make([]byte, ino.Size))
if !bytes.Equal(got, append(first, appended...)) {
t.Error("file data differs from both nodes' writes")
}
}
23 changes: 21 additions & 2 deletions internal/inode/inode.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,14 +209,24 @@ func (t *Table) Read(inodeNum uint32) (*Inode, error) {
// Write writes an inode to disk.
// The inode must have a valid InodeNum set.
func (t *Table) Write(ctx context.Context, inode *Inode) error {
committed, err := t.lockedWrite(ctx, inode)
committed, err := t.lockedWrite(ctx, inode, nil)
if committed != nil {
committed()
}
return err
}

func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) {
// WriteIfUnchanged is Write for an inode read at MetaGeneration gen. If another
// write landed since, it writes nothing and returns ErrTreeConflict.
func (t *Table) WriteIfUnchanged(ctx context.Context, inode *Inode, gen uint64) error {
committed, err := t.lockedWrite(ctx, inode, &gen)
if committed != nil {
committed()
}
return err
}

func (t *Table) lockedWrite(ctx context.Context, inode *Inode, wantGen *uint64) (func(), error) {
if uint64(inode.InodeNum) >= t.totalInodes {
return nil, ErrInodeOutOfRange
}
Expand All @@ -237,6 +247,15 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) {
t.mu.Lock()
defer t.mu.Unlock()

if wantGen != nil {
onDisk, err := t.readInodeUnlocked(inode.InodeNum)
if err != nil {
return nil, err
}
if onDisk.MetaGeneration != *wantGen {
return nil, fmt.Errorf("%w: inode %d was written since it was read", ErrTreeConflict, inode.InodeNum)
}
}
return t.writeWithTree(inode)
}

Expand Down
14 changes: 14 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3426,3 +3426,17 @@ scenarios:
priority: go_only
environments:
go: internal/api/prometheus_test.go::TestMetricsEndpoint_ExportsCapacity

- id: dir-setattr-keeps-mkdir
name: Directory chmod keeps another node's mkdir
description: Node A changes a directory's mode while node B makes a directory in it; the link count and mtime from node B survive
priority: go_only
environments:
go: internal/goclusterfs/setattr_race_test.go::TestDirSetattrKeepsOtherNodesMkdir

- id: file-setattr-keeps-size
name: File chmod keeps another node's size
description: Node A changes a file's mode while node B appends inside the file's allocated block; node B's size and data survive
priority: go_only
environments:
go: internal/goclusterfs/setattr_race_test.go::TestFileSetattrKeepsOtherNodesSize
Loading