diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c3d045..42f52574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- `chmod`, `chown` and `touch` could undo another node's concurrent change to the same inode: on a directory, a create's link count or mtime; on a file, the size an append inside an allocated block set. Directory attribute changes now hold the inode lock from read to write, and file attribute changes retry when the inode changed since they read it. - `chmod` cleared a file's type bits, so the file could no longer be opened ("Is a directory") and fsck reported an invalid file type. `chmod`, `chown` and `touch` now also work on directories and the mount root, which returned "Operation not supported". - The cache summary logged at unmount showed hit rates 100 times too high. - All nodes wrote the same journal region. A restarting node replayed and then reset the journal of nodes still writing, and no node could replay a crashed node's journal. Each node registry slot now owns a 4 MB journal, and a crashed node's journal is replayed through its slot. diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 2166f0ca..d6472361 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -760,6 +760,9 @@ func (f *FS) dropNewInode(ctx context.Context, ino *inode.Inode) { // testHookBeforeMkdirInodeWrite, when set by a test, fails a mkdir's inode write with its error. var testHookBeforeMkdirInodeWrite func() error +// testHookAfterSetattrRead, when set by a test, runs between a setattr's read of the inode and its write. +var testHookAfterSetattrRead func() + func injectedMkdirFailure() error { if testHookBeforeMkdirInodeWrite != nil { return testHookBeforeMkdirInodeWrite() @@ -1112,7 +1115,10 @@ func (f *FS) setattrDir(ctx context.Context, num uint32, in *fuse.SetAttrIn) sys return syscall.EISDIR } now := time.Now().UnixNano() - updated, err := f.updateInode(ctx, num, func(ino *inode.Inode) error { + updated, err := f.updateInodeLocked(ctx, num, func(ino *inode.Inode) error { + if testHookAfterSetattrRead != nil { + testHookAfterSetattrRead() + } if mode, ok := in.GetMode(); ok { ino.Mode = inode.S_IFDIR | uint16(mode&0777) ino.Ctime = now @@ -2659,6 +2665,9 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn if err != nil { return syscall.EIO, nil } + if testHookAfterSetattrRead != nil { + testHookAfterSetattrRead() + } if _, resizing := in.GetSize(); resizing && ino.IsSFBBacked() { if err := f.migrateToExtent(ctx, ino); err != nil { @@ -2672,6 +2681,7 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn return syscall.EIO, nil } } + readGen := ino.MetaGeneration now := time.Now().UnixNano() @@ -2787,7 +2797,8 @@ func (f *File) setattr(ctx context.Context, fh fs.FileHandle, in *fuse.SetAttrIn ino.Mtime = mtime.UnixNano() } - if err := f.fs.inodeTbl.Write(ctx, ino); err != nil { + // Another node's write, such as an append inside an allocated block, may have changed the size since ino was read. + if err := f.fs.inodeTbl.WriteIfUnchanged(ctx, ino, readGen); err != nil { f.fs.extentMgr.DiscardPending(ctx, ino) if errors.Is(err, extent.ErrTreeConflict) { // Nothing was applied, so close the intent before the retry logs a new one. diff --git a/internal/goclusterfs/setattr_race_test.go b/internal/goclusterfs/setattr_race_test.go new file mode 100644 index 00000000..05debcb6 --- /dev/null +++ b/internal/goclusterfs/setattr_race_test.go @@ -0,0 +1,134 @@ +package goclusterfs + +import ( + "bytes" + "context" + "syscall" + "testing" + "time" + + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +// raceSetattr runs other between the next setattr's read of the inode and its +// write. It waits up to a second for other, so other finishes first unless the +// setattr holds the inode lock across its read and write. +func raceSetattr(t *testing.T, other func() syscall.Errno) <-chan syscall.Errno { + t.Helper() + done := make(chan syscall.Errno, 1) + testHookAfterSetattrRead = func() { + testHookAfterSetattrRead = nil + go func() { done <- other() }() + select { + case errno := <-done: + done <- errno + case <-time.After(time.Second): + } + } + t.Cleanup(func() { testHookAfterSetattrRead = nil }) + return done +} + +// withBridge gives fsys a go-fuse bridge without mounting it, so creates can make nodes. +func withBridge(fsys *FS) *FS { + fs.NewNodeFS(fsys, &fs.Options{}) + return fsys +} + +func chmodIn(mode uint32) *fuse.SetAttrIn { + in := &fuse.SetAttrIn{} + in.Valid = fuse.FATTR_MODE + in.Mode = mode + return in +} + +// Node A changes a directory's mode while node B makes a directory in it. +// Node B's link count and mtime must survive node A's write. +func TestDirSetattrKeepsOtherNodesMkdir(t *testing.T) { + path := newFormattedImage(t) + nodeA, nodeB := withBridge(openFormattedFS(t, path)), withBridge(openFormattedFS(t, path)) + ctx := context.Background() + + var out fuse.EntryOut + if _, errno := nodeA.Mkdir(ctx, "d", 0o755, &out); errno != 0 { + t.Fatalf("mkdir d: %v", errno) + } + num := uint32(out.Ino) + before, err := nodeA.inodeTbl.Read(num) + if err != nil { + t.Fatal(err) + } + + made := raceSetattr(t, func() syscall.Errno { + _, errno := nodeB.makeDir(ctx, &nodeB.Inode, num, "sub", 0o755, &fuse.EntryOut{}) + return errno + }) + if errno := (&Dir{fs: nodeA, inodeNum: num}).Setattr(ctx, nil, chmodIn(0o700), &fuse.AttrOut{}); errno != 0 { + t.Fatalf("chmod d on node A: %v", errno) + } + if errno := <-made; errno != 0 { + t.Fatalf("mkdir d/sub on node B: %v", errno) + } + + after, err := nodeA.inodeTbl.Read(num) + if err != nil { + t.Fatal(err) + } + if after.Mode&0o777 != 0o700 { + t.Errorf("mode %o after chmod 700", after.Mode&0o777) + } + if after.LinkCount != before.LinkCount+1 { + t.Errorf("link count %d after node B's mkdir, want %d", after.LinkCount, before.LinkCount+1) + } + if after.Mtime <= before.Mtime { + t.Errorf("mtime %d after node B's mkdir, not later than %d", after.Mtime, before.Mtime) + } +} + +// Node A changes a file's mode while node B appends inside the file's only +// block, which changes the size but not the extents. Node A must keep that size. +func TestFileSetattrKeepsOtherNodesSize(t *testing.T) { + path := newFormattedImage(t) + nodeA, nodeB := openFormattedFS(t, path), openFormattedFS(t, path) + ctx := context.Background() + + fA := newRegularFile(t, nodeA) + first := pattern(1, 100) + writeAt(t, fA, first, 0) + fB := &File{fs: nodeB, inodeNum: fA.inodeNum} + appended := pattern(2, 100) + + wrote := raceSetattr(t, func() syscall.Errno { + n, errno := fB.Write(ctx, nil, appended, int64(len(first))) + if errno == 0 && int(n) != len(appended) { + return syscall.EIO + } + return errno + }) + if errno := fA.Setattr(ctx, nil, chmodIn(0o600), &fuse.AttrOut{}); errno != 0 { + t.Fatalf("chmod on node A: %v", errno) + } + if errno := <-wrote; errno != 0 { + t.Fatalf("append on node B: %v", errno) + } + + ino, err := nodeA.inodeTbl.Read(fA.inodeNum) + if err != nil { + t.Fatal(err) + } + if ino.Mode&0o777 != 0o600 { + t.Errorf("mode %o after chmod 600", ino.Mode&0o777) + } + if want := uint64(len(first) + len(appended)); ino.Size != want { + t.Fatalf("size %d after node B's append, want %d", ino.Size, want) + } + res, errno := fA.Read(ctx, nil, make([]byte, ino.Size), 0) + if errno != 0 { + t.Fatalf("read: %v", errno) + } + got, _ := res.Bytes(make([]byte, ino.Size)) + if !bytes.Equal(got, append(first, appended...)) { + t.Error("file data differs from both nodes' writes") + } +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 6171f79c..dff7c300 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -209,14 +209,24 @@ func (t *Table) Read(inodeNum uint32) (*Inode, error) { // Write writes an inode to disk. // The inode must have a valid InodeNum set. func (t *Table) Write(ctx context.Context, inode *Inode) error { - committed, err := t.lockedWrite(ctx, inode) + committed, err := t.lockedWrite(ctx, inode, nil) if committed != nil { committed() } return err } -func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { +// WriteIfUnchanged is Write for an inode read at MetaGeneration gen. If another +// write landed since, it writes nothing and returns ErrTreeConflict. +func (t *Table) WriteIfUnchanged(ctx context.Context, inode *Inode, gen uint64) error { + committed, err := t.lockedWrite(ctx, inode, &gen) + if committed != nil { + committed() + } + return err +} + +func (t *Table) lockedWrite(ctx context.Context, inode *Inode, wantGen *uint64) (func(), error) { if uint64(inode.InodeNum) >= t.totalInodes { return nil, ErrInodeOutOfRange } @@ -237,6 +247,15 @@ func (t *Table) lockedWrite(ctx context.Context, inode *Inode) (func(), error) { t.mu.Lock() defer t.mu.Unlock() + if wantGen != nil { + onDisk, err := t.readInodeUnlocked(inode.InodeNum) + if err != nil { + return nil, err + } + if onDisk.MetaGeneration != *wantGen { + return nil, fmt.Errorf("%w: inode %d was written since it was read", ErrTreeConflict, inode.InodeNum) + } + } return t.writeWithTree(inode) } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index a3ef1f98..bf239ed7 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3426,3 +3426,17 @@ scenarios: priority: go_only environments: go: internal/api/prometheus_test.go::TestMetricsEndpoint_ExportsCapacity + + - id: dir-setattr-keeps-mkdir + name: Directory chmod keeps another node's mkdir + description: Node A changes a directory's mode while node B makes a directory in it; the link count and mtime from node B survive + priority: go_only + environments: + go: internal/goclusterfs/setattr_race_test.go::TestDirSetattrKeepsOtherNodesMkdir + + - id: file-setattr-keeps-size + name: File chmod keeps another node's size + description: Node A changes a file's mode while node B appends inside the file's allocated block; node B's size and data survive + priority: go_only + environments: + go: internal/goclusterfs/setattr_race_test.go::TestFileSetattrKeepsOtherNodesSize