From 410e6a361d5bceecc01276889381c6b4dcc1ffdf Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 10:56:59 -0600 Subject: [PATCH 1/5] test(lock): reproduce a wait cycle between inode and extent locks Two nodes each hold an inode lock and then want an extent lock that hashes to the record of the other node's inode. Both follow type order, and both wait until the timeout. Co-Authored-By: Claude Opus 5.5 --- internal/lock/type_records_test.go | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 internal/lock/type_records_test.go diff --git a/internal/lock/type_records_test.go b/internal/lock/type_records_test.go new file mode 100644 index 00000000..ea65561d --- /dev/null +++ b/internal/lock/type_records_test.go @@ -0,0 +1,83 @@ +package lock + +import ( + "fmt" + "sync" + "testing" + "time" +) + +// crossedKeys finds inodes a, c and extents b, d with b in c's record and d in a's: a wait cycle between two nodes. +func crossedKeys(m *Manager) (a, b, c, d uint64, ok bool) { + extentIn := map[uint64]uint64{} + for id := uint64(0); id < 1<<16; id++ { + r := m.hashToRecord(LockTypeExtent, id) + if _, seen := extentIn[r]; !seen { + extentIn[r] = id + } + } + extentFor := func(inode uint64) (uint64, bool) { + e, ok := extentIn[m.hashToRecord(LockTypeInode, inode)] + return e, ok + } + + for a = 1; a < 1<<16; a++ { + if d, ok = extentFor(a); ok { + break + } + } + if !ok { + return 0, 0, 0, 0, false + } + for c = a + 1; c < 1<<16; c++ { + if m.hashToRecord(LockTypeInode, c) == m.hashToRecord(LockTypeInode, a) { + continue + } + if b, ok = extentFor(c); ok { + return a, b, c, d, true + } + } + return 0, 0, 0, 0, false +} + +func TestInodeAndExtentLocksDoNotWaitInACycle(t *testing.T) { + const region = 256 * RowSize + dev, cleanup := createTestDevice(t, region) + defer cleanup() + x := newTestNode(dev, 0, region, 1) + y := newTestNode(dev, 0, region, 2) + + a, b, c, d, ok := crossedKeys(x) + if !ok { + t.Log("no extent lock shares a record with an inode lock") + return + } + + var held sync.WaitGroup + held.Add(2) + write := func(m *Manager, inode, extent uint64) error { + err := acquireWithin(m, time.Second, LockTypeInode, inode, LockModeExclusive) + held.Done() + if err != nil { + return fmt.Errorf("inode %d: %w", inode, err) + } + defer m.Release(LockTypeInode, inode) + held.Wait() + if err := acquireWithin(m, 2*time.Second, LockTypeExtent, extent, LockModeExclusive); err != nil { + return fmt.Errorf("extent %d while holding inode %d: %w", extent, inode, err) + } + return m.Release(LockTypeExtent, extent) + } + + errs := make([]error, 2) + var done sync.WaitGroup + done.Add(2) + go func() { defer done.Done(); errs[0] = write(x, a, b) }() + go func() { defer done.Done(); errs[1] = write(y, c, d) }() + done.Wait() + for i, err := range errs { + if err != nil { + t.Errorf("node %d: %v", i+1, err) + } + } +} From d1437627b52d416cdf423ee531aced4214b98530 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 11:15:28 -0600 Subject: [PATCH 2/5] fix(lock): give each lock type its own records Two nodes could each hold an inode lock and wait for an extent lock in the record the other's inode lock held, then fail with EIO after the timeout. Type order did not prevent it, because locks of different types shared records. Each type now has its own records, laid out in type order and sized by how many keys of the type are held at once. checkLockOrder also refuses a lock of a type the goroutine holds in a lower record, so a goroutine's records only go up and waits between nodes cannot form a cycle. No production path holds two locks of one type at once today. Lock regions smaller than a volume's, which only tests use, still hash over all records. The sector lock test from #81 becomes a test that no type waits on records another type uses. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/OPERATIONS.md | 2 +- internal/lock/lock.go | 67 ++++++++++++++-------- internal/lock/sector_test.go | 35 ------------ internal/lock/type_records_test.go | 90 ++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 29 ++++++++-- 6 files changed, 163 insertions(+), 63 deletions(-) delete mode 100644 internal/lock/sector_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b2bdf1..27f475ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. - - This version places lock records differently (inode table sector locks have their own records), so old and new binaries on one device do not exclude each other. + - This version places lock records differently (each lock type has its own records), so old and new binaries on one device do not exclude each other. - Directory entries now fill all 4 KB chunks of a directory block. When an older binary removes the first entry of a later chunk, it grows the last entry of the chunk before it across the boundary. This version then stops listing that entry, and fsck reports the directory block. - Run `goclusterfs-fsck -repair` before the first mount after upgrading an image that has snapshots or clones. Unmount the device on every node first. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. Until `goclusterfs-fsck -repair` counts them, this version never frees such blocks, so deleting those files leaks their space, and it refuses to snapshot or clone them (`ENOTSUP`). In-place writes to an old clone still change its source. @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - A node could serve a stale cached inode after another node freed and reused its inode number. Inode generations now survive free and reuse, and opening a file reloads its inode from disk. - Journal recovery no longer panics, loops forever, or allocates gigabytes on a corrupt journal. It now rejects short entries and bad ring geometry as corrupt. - Lock-order violations when several nodes create files and directories at the same time. +- Two nodes could each hold a lock and wait for the other's, then fail with `EIO` after 30 seconds, when a write's inode and block locks mapped to records the other node held. Each lock type now has its own lock records. - Inode allocation races between nodes; allocation now retries with jittered backoff. - Directory operations now update caches and send broadcast invalidations. - Docker and VM test suites now run on fresh hosts. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 435e5751..e382207a 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -255,7 +255,7 @@ Lock slot reads and writes do not take the device's per-region I/O lock, so a no **Read locks are not renewed.** A node whose overlapping lookups keep a directory read-locked for more than 15 seconds can lose that lock to another node's writer. Its readers may then see a half-written directory. Two writers still cannot overlap. -**Locks share records.** The lock region has 256 records, and every lock maps to one of them, so unrelated files and the block bitmap can share a record. A node holding one of those locks blocks other nodes from all of them. If two nodes each hold one record and wait for the other's, both wait until one record is 15 seconds old; the takeover then lets both proceed, with the same risk as a stall. This needs nested locks whose records cross, roughly 1 in 65,000 pairs of overlapping operations. +**Locks of one type share records.** Each lock type has its own records of the 256: 128 for inodes, 97 for block extents, 16 for inode table sectors, 8 for SFB containers, and one for each other type. Two files' inode locks can map to one record, and a node holding one of them blocks other nodes from both. A lock never waits on a record another type uses. A node takes locks in type order and, within a type, in record order, so two nodes cannot each wait for a record the other holds. **Safe operating limits:** diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 18c18a2d..2b4e41d6 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -19,6 +19,10 @@ // Level 9: Extent (by block) // Level 10: InodeSector (by inode table sector), a leaf: held only around // one sector's read-modify-write, with no other lock taken while held +// +// Each type has its own records, laid out in type order, and locks of one type +// are taken in record order. A goroutine's records then only go up, so waits +// between nodes cannot form a cycle. package lock import ( @@ -277,6 +281,31 @@ func (m *Manager) Stats() Stats { const numLockTypes = int(LockTypeInodeSector) + 1 +// layoutRows is how many records a volume's lock region has; typeRows divides them among the types. +const layoutRows = 256 + +// typeRows is each type's share of the records, by how many keys of the type nodes hold at once. +var typeRows = [numLockTypes]uint64{ + LockTypeVolume: 1, + LockTypeSnapshotReg: 1, + LockTypeSnapshot: 1, + LockTypeJournal: 1, + LockTypeRefCount: 1, + LockTypeLockRegion: 1, + LockTypeBlockBitmap: 1, + LockTypeSFBContainer: 8, + LockTypeInode: 128, + LockTypeExtent: 97, + LockTypeInodeSector: 16, +} + +var typeFirstRow = func() (first [numLockTypes]uint64) { + for t := 1; t < numLockTypes; t++ { + first[t] = first[t-1] + typeRows[t-1] + } + return first +}() + // acquireCounts says how one lock type's acquires on this node were served: // a key this node already held, a join of a held record, a cached record, a // claim won on disk, and tries that had to wait. @@ -359,7 +388,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin m.countAcquire(lockType, func(c *acquireCounts) *atomic.Uint64 { return &c.held }) return nil } - if err := m.checkLockOrder(lockType); err != nil { + if err := m.checkLockOrder(key, recordID, gid); err != nil { m.mu.Unlock() return err } @@ -684,18 +713,19 @@ func (m *Manager) ReleaseCached() { wg.Wait() } -// checkLockOrder ensures we're acquiring locks in hierarchy order. -// Only checks against locks held by the CURRENT goroutine to avoid -// false positives when different goroutines hold different locks concurrently. -func (m *Manager) checkLockOrder(newType LockType) error { - gid := getGoroutineID() - for key, held := range m.heldLocks { - // Only check order for locks held by THIS goroutine - if held.goroutines != nil && held.goroutines[gid] > 0 { - if key.Type > newType { - return fmt.Errorf("%w: cannot acquire %v while holding %v", - ErrDeadlockRisk, newType, key.Type) - } +// checkLockOrder refuses a lock below one this goroutine holds: a lower type, or a lower record of the same type. +func (m *Manager) checkLockOrder(key lockKey, recordID, gid uint64) error { + for heldKey, held := range m.heldLocks { + if heldKey == key || held.goroutines[gid] == 0 { + continue + } + if heldKey.Type > key.Type { + return fmt.Errorf("%w: cannot acquire %v while holding %v", + ErrDeadlockRisk, key.Type, heldKey.Type) + } + if heldKey.Type == key.Type && held.RecordID > recordID { + return fmt.Errorf("%w: cannot acquire type %v in record %d while holding its record %d; take locks of one type in record order", + ErrDeadlockRisk, key.Type, recordID, held.RecordID) } } return nil @@ -710,18 +740,11 @@ func (m *Manager) hashToRecord(lockType LockType, resourceID uint64) uint64 { hash *= 0xFF51AFD7ED558CCD hash ^= hash >> 33 - // Inode table sector locks get their own records. A sector lock is a leaf, - // so its holder never waits on another lock, and no cycle of waits between - // nodes can pass through a record only sector locks use. records := m.size / RowSize - leaf := records / 16 - if leaf == 0 { + if records < layoutRows || int(lockType) >= numLockTypes { return hash % records } - if lockType == LockTypeInodeSector { - return records - leaf + hash%leaf - } - return hash % (records - leaf) + return typeFirstRow[lockType] + hash%typeRows[lockType] } // SameRecord reports whether two locks map to one record, so one goroutine holding both holds the record in the stronger mode. diff --git a/internal/lock/sector_test.go b/internal/lock/sector_test.go deleted file mode 100644 index 0f930d82..00000000 --- a/internal/lock/sector_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package lock - -import ( - "testing" - "time" -) - -// TestSectorLocksDoNotWaitOnOtherLocks has one node hold inode locks spread -// over the lock records, then another node take inode table sector locks. Sector -// locks have records of their own, so none may wait: a sector lock waiting on a -// record held for another lock type could close a cycle of waits between nodes. -func TestSectorLocksDoNotWaitOnOtherLocks(t *testing.T) { - const region = 256 * RowSize - dev, cleanup := createTestDevice(t, region) - defer cleanup() - a := newTestNode(dev, 0, region, 1) - b := newTestNode(dev, 0, region, 2) - - const inodes, sectors = 1024, 64 - for id := uint64(0); id < inodes; id++ { - requireAcquired(t, b, LockTypeInode, id, LockModeExclusive) - } - defer func() { - for id := uint64(0); id < inodes; id++ { - b.Release(LockTypeInode, id) - } - }() - - for s := uint64(0); s < sectors; s++ { - if err := acquireWithin(a, 200*time.Millisecond, LockTypeInodeSector, s, LockModeExclusive); err != nil { - t.Fatalf("sector %d lock waited on a record node B holds for inode locks: %v", s, err) - } - a.Release(LockTypeInodeSector, s) - } -} diff --git a/internal/lock/type_records_test.go b/internal/lock/type_records_test.go index ea65561d..050e2d45 100644 --- a/internal/lock/type_records_test.go +++ b/internal/lock/type_records_test.go @@ -1,10 +1,15 @@ package lock import ( + "errors" "fmt" + "maps" + "slices" "sync" "testing" "time" + + "github.com/supporttools/GoClusterFS/internal/superblock" ) // crossedKeys finds inodes a, c and extents b, d with b in c's record and d in a's: a wait cycle between two nodes. @@ -81,3 +86,88 @@ func TestInodeAndExtentLocksDoNotWaitInACycle(t *testing.T) { } } } + +func TestLockTypesHaveTheirOwnRecords(t *testing.T) { + var rows uint64 + for _, n := range typeRows { + rows += n + } + if rows != layoutRows || layoutRows*RowSize != superblock.LockRegionSize { + t.Fatalf("types share %d rows of %d bytes, a volume has %d bytes of lock records", rows, RowSize, superblock.LockRegionSize) + } + + m := &Manager{size: layoutRows * RowSize} + owner := map[uint64]LockType{} + for lt := LockType(0); int(lt) < numLockTypes; lt++ { + for id := uint64(0); id < 1<<12; id++ { + r := m.hashToRecord(lt, id) + if other, ok := owner[r]; ok && other != lt { + t.Fatalf("type %d resource %d maps to record %d, which type %d also uses", lt, id, r, other) + } + owner[r] = lt + } + } +} + +// Node B holds a lock in every inode record; node A's locks of other types must not wait on any of them. +func TestLocksDoNotWaitOnRecordsOfOtherTypes(t *testing.T) { + const region = layoutRows * RowSize + dev, cleanup := createTestDevice(t, region) + defer cleanup() + a := newTestNode(dev, 0, region, 1) + b := newTestNode(dev, 0, region, 2) + + inodeIn := map[uint64]uint64{} + for id := uint64(0); len(inodeIn) < int(typeRows[LockTypeInode]); id++ { + r := b.hashToRecord(LockTypeInode, id) + if _, ok := inodeIn[r]; !ok { + inodeIn[r] = id + } + } + rows := slices.Sorted(maps.Keys(inodeIn)) + for _, r := range rows { + requireAcquired(t, b, LockTypeInode, inodeIn[r], LockModeExclusive) + } + defer func() { + for _, r := range rows { + b.Release(LockTypeInode, inodeIn[r]) + } + }() + + for lt := LockType(0); int(lt) < numLockTypes; lt++ { + if lt == LockTypeInode { + continue + } + for id := uint64(0); id < 64; id++ { + if err := acquireWithin(a, 200*time.Millisecond, lt, id, LockModeExclusive); err != nil { + t.Fatalf("type %d resource %d waited on a record node B holds for inode locks: %v", lt, id, err) + } + a.Release(lt, id) + } + } +} + +func TestLocksOfOneTypeAreTakenInRecordOrder(t *testing.T) { + const region = layoutRows * RowSize + dev, cleanup := createTestDevice(t, region) + defer cleanup() + m := newTestNode(dev, 0, region, 1) + + low, high := uint64(1), uint64(2) + for m.hashToRecord(LockTypeInode, high) == m.hashToRecord(LockTypeInode, low) { + high++ + } + if m.hashToRecord(LockTypeInode, low) > m.hashToRecord(LockTypeInode, high) { + low, high = high, low + } + requireAcquired(t, m, LockTypeInode, high, LockModeExclusive) + if err := acquireWithin(m, time.Second, LockTypeInode, low, LockModeExclusive); !errors.Is(err, ErrDeadlockRisk) { + t.Fatalf("inode %d below held inode %d: err = %v, want ErrDeadlockRisk", low, high, err) + } + m.Release(LockTypeInode, high) + + requireAcquired(t, m, LockTypeInode, low, LockModeExclusive) + requireAcquired(t, m, LockTypeInode, high, LockModeExclusive) + m.Release(LockTypeInode, high) + m.Release(LockTypeInode, low) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 6933a5ce..85c146f4 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1925,12 +1925,33 @@ scenarios: environments: go: internal/inode/sector_lock_test.go::TestNeighborInodeWritesFromTwoNodes - - id: sector-locks-own-records - name: Sector locks never wait on other lock types - description: Inode table sector locks map to reserved lock records, so a node holding inode locks never blocks another node's sector lock + - id: lock-type-wait-cycle + name: Inode and extent locks do not wait in a cycle + description: Two nodes each hold an inode lock and want an extent lock that would share the other's inode record; neither waits on the other priority: go_only environments: - go: internal/lock/sector_test.go::TestSectorLocksDoNotWaitOnOtherLocks + go: internal/lock/type_records_test.go::TestInodeAndExtentLocksDoNotWaitInACycle + + - id: lock-types-own-records + name: Each lock type has its own records + description: The per-type record counts fill a volume's lock region exactly, and no two types map a resource to the same record + priority: go_only + environments: + go: internal/lock/type_records_test.go::TestLockTypesHaveTheirOwnRecords + + - id: locks-no-wait-other-types + name: Locks never wait on records of another type + description: With one node holding a lock in every inode record, another node takes locks of every other type without waiting + priority: go_only + environments: + go: internal/lock/type_records_test.go::TestLocksDoNotWaitOnRecordsOfOtherTypes + + - id: lock-record-order + name: Locks of one type are taken in record order + description: A goroutine holding a lock is refused a lock of the same type in a lower record, and may take one in a higher record + priority: go_only + environments: + go: internal/lock/type_records_test.go::TestLocksOfOneTypeAreTakenInRecordOrder - id: create-stuck-inode-lock-eio name: Create fails with EIO when the new inode's lock is never freed From cf7d17bc673b6538d1db400a1f031c5d41c4f892 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 11:21:07 -0600 Subject: [PATCH 3/5] fix(superblock): refuse volumes whose lock records hash over all types Nodes that map locks to records differently do not exclude each other. New volumes set RequiredTypedLockRecords: older builds refuse them as an unknown required flag, and mount refuses volumes without it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +-- internal/superblock/io.go | 2 +- internal/superblock/superblock.go | 11 +++++++++-- internal/superblock/superblock_test.go | 13 ++++++++++--- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27f475ca..f724d574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - The on-disk format is now version 3. Mount refuses devices formatted by earlier builds. Version 1 kept one journal for all nodes. Version 2 had room for 4096 inodes, and on devices over 128 GB its refcounts overwrote the snapshot registry and then data blocks. Copy the data off with the old build, reformat with `-format`, and copy it back. - Devices must be at least 327 MB: the layout reserves 256 MB for 64 node journals of 4 MB and 64 MB for lock records. -- Mount also refuses version 3 devices formatted before lock records had one claim slot per node. Copy the data off with the build that formatted them and reformat. +- Mount also refuses version 3 devices formatted before lock records had one claim slot per node, or before each lock type had its own lock records. Older builds refuse devices this version formats. Copy the data off with the build that formatted them and reformat. - Node clocks must agree within a few seconds. See the node requirements in `docs/OPERATIONS.md`. - Upgrade all nodes together: unmount on every node, upgrade, then mount again. Don't mount one shared device with a mix of old and new binaries. - Older binaries reset an inode's generation counters to 0 when they free it, so other nodes can serve stale data after the inode number is reused. - Older binaries zero the extent-tree generation (inode bytes 188–195) when they rewrite an inode. Newer binaries then cannot read that file if it has more than two extents, because such files use an extent tree. - - This version places lock records differently (each lock type has its own records), so old and new binaries on one device do not exclude each other. - Directory entries now fill all 4 KB chunks of a directory block. When an older binary removes the first entry of a later chunk, it grows the last entry of the chunk before it across the boundary. This version then stops listing that entry, and fsck reports the directory block. - Run `goclusterfs-fsck -repair` before the first mount after upgrading an image that has snapshots or clones. Unmount the device on every node first. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. Until `goclusterfs-fsck -repair` counts them, this version never frees such blocks, so deleting those files leaks their space, and it refuses to snapshot or clone them (`ENOTSUP`). In-place writes to an old clone still change its source. diff --git a/internal/superblock/io.go b/internal/superblock/io.go index f1958dcd..e7fcafe5 100644 --- a/internal/superblock/io.go +++ b/internal/superblock/io.go @@ -266,7 +266,7 @@ func New(deviceSize int64, bytesPerInode uint64) *Superblock { sb := &Superblock{ Version: CurrentVersion, FeatureFlags: FeatureSparseFiles | FeatureLargeFiles, - RequiredFlags: RequiredJournal, + RequiredFlags: RequiredJournal | RequiredTypedLockRecords, // Geometry BlockSize: storage.DefaultBlockSize, // 1MB diff --git a/internal/superblock/superblock.go b/internal/superblock/superblock.go index b29f4f48..0695443b 100644 --- a/internal/superblock/superblock.go +++ b/internal/superblock/superblock.go @@ -40,11 +40,14 @@ const ( RequiredSnapshots uint64 = 1 << 1 // Snapshot structures present RequiredEncryption uint64 = 1 << 2 // Encryption enabled (future) RequiredCompressed uint64 = 1 << 3 // Compression enabled (future) + + // RequiredTypedLockRecords means each lock type has its own lock records; builds without it hash every type over all of them. + RequiredTypedLockRecords uint64 = 1 << 4 ) // FlagsWeSupport defines which required flags this version understands. // Update this as new features are implemented. -var FlagsWeSupport uint64 = RequiredJournal | RequiredSnapshots +var FlagsWeSupport uint64 = RequiredJournal | RequiredSnapshots | RequiredTypedLockRecords // Errors var ( @@ -129,7 +132,11 @@ func (sb *Superblock) CanMount() error { return fmt.Errorf("%w: unknown required flags 0x%x", ErrUnknownRequired, unknownFlags) } - + if sb.RequiredFlags&RequiredTypedLockRecords == 0 { + return fmt.Errorf("%w: the volume was formatted before each lock type had its own lock records, so its nodes "+ + "would not exclude this version's. Copy the data off with the version that formatted it, then reformat with -format", + ErrUnsupportedLayout) + } return nil } diff --git a/internal/superblock/superblock_test.go b/internal/superblock/superblock_test.go index 5204f540..2c5d60d7 100644 --- a/internal/superblock/superblock_test.go +++ b/internal/superblock/superblock_test.go @@ -138,7 +138,7 @@ func TestDeserializeBufferTooSmall(t *testing.T) { func TestCanMountCurrentVersion(t *testing.T) { sb := &Superblock{ Version: CurrentVersion, - RequiredFlags: RequiredJournal, // We support this + RequiredFlags: RequiredJournal | RequiredTypedLockRecords, } if err := sb.CanMount(); err != nil { @@ -150,7 +150,7 @@ func TestCanMountOlderVersion(t *testing.T) { // Test that we accept older versions within range sb := &Superblock{ Version: MinSupportedVersion, - RequiredFlags: 0, // No required flags + RequiredFlags: RequiredTypedLockRecords, } if err := sb.CanMount(); err != nil { @@ -203,7 +203,7 @@ func TestCanMountWithOptionalFeatures(t *testing.T) { sb := &Superblock{ Version: CurrentVersion, FeatureFlags: 0xFFFFFFFF, // All optional flags set - RequiredFlags: RequiredJournal, + RequiredFlags: RequiredJournal | RequiredTypedLockRecords, } if err := sb.CanMount(); err != nil { @@ -211,6 +211,13 @@ func TestCanMountWithOptionalFeatures(t *testing.T) { } } +func TestCanMountRefusesHashedLockRecords(t *testing.T) { + sb := &Superblock{Version: CurrentVersion, RequiredFlags: RequiredJournal} + if err := sb.CanMount(); !errors.Is(err, ErrUnsupportedLayout) || !strings.Contains(err.Error(), "reformat with -format") { + t.Errorf("CanMount of a volume without per-type lock records: err = %v, want ErrUnsupportedLayout saying to reformat", err) + } +} + func TestNewSuperblock(t *testing.T) { deviceSize := int64(10 << 30) // 10GB sb := New(deviceSize, 0) From 8bbd7a85a8b3f8d5c51f46c433347dab302d595c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 11:39:28 -0600 Subject: [PATCH 4/5] docs(lock): note that adopted records can still close a wait cycle A replaying node holds a failed node's adopted records until the replay ends and takes its own locks in journal order, so the record order rule does not cover waits on those records. Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 2 +- internal/lock/lock.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index e382207a..f99d35f4 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -255,7 +255,7 @@ Lock slot reads and writes do not take the device's per-region I/O lock, so a no **Read locks are not renewed.** A node whose overlapping lookups keep a directory read-locked for more than 15 seconds can lose that lock to another node's writer. Its readers may then see a half-written directory. Two writers still cannot overlap. -**Locks of one type share records.** Each lock type has its own records of the 256: 128 for inodes, 97 for block extents, 16 for inode table sectors, 8 for SFB containers, and one for each other type. Two files' inode locks can map to one record, and a node holding one of them blocks other nodes from both. A lock never waits on a record another type uses. A node takes locks in type order and, within a type, in record order, so two nodes cannot each wait for a record the other holds. +**Locks of one type share records.** Each lock type has its own records of the 256: 128 for inodes, 97 for block extents, 16 for inode table sectors, 8 for SFB containers, and one for each other type. Two files' inode locks can map to one record, and a node holding one of them blocks other nodes from both. A lock never waits on a record another type uses. A node takes locks in type order and, within a type, in record order, so two nodes cannot each wait for a record the other holds. The exception is a replay of a failed node's journal: the replaying node keeps the failed node's records until the replay ends, so a node that holds a lock the replay needs and waits on one of those records waits until one side gives up after 30 seconds. **Safe operating limits:** diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 2b4e41d6..49b2a436 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -22,7 +22,9 @@ // // Each type has its own records, laid out in type order, and locks of one type // are taken in record order. A goroutine's records then only go up, so waits -// between nodes cannot form a cycle. +// between nodes cannot form a cycle, except through records a node adopts to +// replay a failed node's journal: the replay holds them until it ends and takes +// its own locks in journal order. package lock import ( @@ -378,6 +380,9 @@ func (m *Manager) LogStats() { // Resources that hash to the same record share it: a node holding the record // for one resource excludes other nodes from every resource in that record. func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uint64, mode LockMode) error { + if int(lockType) >= numLockTypes { + return fmt.Errorf("lock: unknown lock type %d", lockType) + } key := lockKey{Type: lockType, ResourceID: resourceID} recordID := m.hashToRecord(lockType, resourceID) gid := getGoroutineID() From 731e799dfa014668594d6d88b452e5272aefca54 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 11:39:28 -0600 Subject: [PATCH 5/5] fix(fsck): refuse volumes without per-type lock records, as mount does fsck phase 1 only rejected unknown required flags, so it checked a volume mount refuses and then told fsck slot release to "run fsck first". Both now use superblock.CheckRequiredFlags. Acquire also refuses a lock type outside the record layout instead of hashing it over all records. Co-Authored-By: Claude Opus 5.5 --- internal/fsck/fsck_test.go | 23 +++++++++++++++++++++++ internal/fsck/phase1_superblock.go | 10 +++------- internal/lock/type_records_test.go | 10 ++++++++++ internal/superblock/superblock.go | 15 ++++++++------- tests/test-manifest.yaml | 21 +++++++++++++++++++++ 5 files changed, 65 insertions(+), 14 deletions(-) diff --git a/internal/fsck/fsck_test.go b/internal/fsck/fsck_test.go index 6fe79532..7f1a3482 100644 --- a/internal/fsck/fsck_test.go +++ b/internal/fsck/fsck_test.go @@ -565,3 +565,26 @@ func TestFsckExtentTree(t *testing.T) { t.Fatal("data extent pointing at another inode's tree node not reported") } } + +func TestFsckRefusesHashedLockRecords(t *testing.T) { + dev, cleanup := testDevice(t, 20) + defer cleanup() + sb := superblock.New(dev.Size(), 0) + sb.TotalBlocks, sb.FreeBlocks = 9, 9 + sb.RequiredFlags &^= superblock.RequiredTypedLockRecords + if _, err := dev.WriteAt(sb.Serialize(), 0); err != nil { + t.Fatal(err) + } + + opts := fsck.DefaultOptions() + opts.Output = &bytes.Buffer{} + result := fsck.New(dev, opts).Check() + found := false + for _, p := range result.Problems { + found = found || p.Phase == fsck.PhaseSuperblock && p.Severity == fsck.Fatal && strings.Contains(p.Message, "reformat with -format") + } + if !result.Aborted || !found { + t.Fatalf("fsck of a volume without per-type lock records: aborted=%v, problems %v; want a fatal superblock problem saying to reformat", + result.Aborted, result.Problems) + } +} diff --git a/internal/fsck/phase1_superblock.go b/internal/fsck/phase1_superblock.go index d5eee496..23a68d70 100644 --- a/internal/fsck/phase1_superblock.go +++ b/internal/fsck/phase1_superblock.go @@ -56,13 +56,9 @@ func (c *Checker) checkSuperblock() error { } c.logf(" Version: %d (OK)\n", sb.Version) - // Check required flags - unknownFlags := sb.RequiredFlags &^ superblock.FlagsWeSupport - if unknownFlags != 0 { - c.addProblem(PhaseSuperblock, Fatal, "offset 40", - fmt.Sprintf("unknown required flags: 0x%X", unknownFlags), - false) - return fmt.Errorf("unknown required flags") + if err := superblock.CheckRequiredFlags(sb.RequiredFlags); err != nil { + c.addProblem(PhaseSuperblock, Fatal, "offset 40", err.Error(), false) + return fmt.Errorf("unsupported required flags") } c.logf(" Required Flags: 0x%X (OK)\n", sb.RequiredFlags) diff --git a/internal/lock/type_records_test.go b/internal/lock/type_records_test.go index 050e2d45..0153cc67 100644 --- a/internal/lock/type_records_test.go +++ b/internal/lock/type_records_test.go @@ -171,3 +171,13 @@ func TestLocksOfOneTypeAreTakenInRecordOrder(t *testing.T) { m.Release(LockTypeInode, high) m.Release(LockTypeInode, low) } + +func TestAcquireRefusesUnknownLockType(t *testing.T) { + const region = layoutRows * RowSize + dev, cleanup := createTestDevice(t, region) + defer cleanup() + m := newTestNode(dev, 0, region, 1) + if err := acquireWithin(m, time.Second, LockType(numLockTypes), 1, LockModeExclusive); err == nil { + t.Fatal("Acquire of a lock type outside the record layout succeeded") + } +} diff --git a/internal/superblock/superblock.go b/internal/superblock/superblock.go index 0695443b..71321477 100644 --- a/internal/superblock/superblock.go +++ b/internal/superblock/superblock.go @@ -125,14 +125,15 @@ func (sb *Superblock) CanMount() error { return err } - // Check for required features we don't understand - // &^ is "AND NOT" - gives us bits set in RequiredFlags but not in FlagsWeSupport - unknownFlags := sb.RequiredFlags &^ FlagsWeSupport - if unknownFlags != 0 { - return fmt.Errorf("%w: unknown required flags 0x%x", - ErrUnknownRequired, unknownFlags) + return CheckRequiredFlags(sb.RequiredFlags) +} + +// CheckRequiredFlags refuses required flags this build does not know, and volumes missing one it needs. +func CheckRequiredFlags(flags uint64) error { + if unknown := flags &^ FlagsWeSupport; unknown != 0 { + return fmt.Errorf("%w: unknown required flags 0x%x", ErrUnknownRequired, unknown) } - if sb.RequiredFlags&RequiredTypedLockRecords == 0 { + if flags&RequiredTypedLockRecords == 0 { return fmt.Errorf("%w: the volume was formatted before each lock type had its own lock records, so its nodes "+ "would not exclude this version's. Copy the data off with the version that formatted it, then reformat with -format", ErrUnsupportedLayout) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 85c146f4..bcfec3f6 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1953,6 +1953,27 @@ scenarios: environments: go: internal/lock/type_records_test.go::TestLocksOfOneTypeAreTakenInRecordOrder + - id: lock-unknown-type + name: Acquire refuses a lock type outside the record layout + description: A lock type with no records of its own is refused instead of hashed over all records + priority: go_only + environments: + go: internal/lock/type_records_test.go::TestAcquireRefusesUnknownLockType + + - id: mount-refuses-hashed-lock-records + name: Mount refuses volumes without per-type lock records + description: A superblock without RequiredTypedLockRecords fails CanMount with advice to reformat + priority: go_only + environments: + go: internal/superblock/superblock_test.go::TestCanMountRefusesHashedLockRecords + + - id: fsck-refuses-hashed-lock-records + name: fsck refuses volumes without per-type lock records + description: fsck phase 1 reports a missing RequiredTypedLockRecords flag as fatal, with advice to reformat + priority: go_only + environments: + go: internal/fsck/fsck_test.go::TestFsckRefusesHashedLockRecords + - id: create-stuck-inode-lock-eio name: Create fails with EIO when the new inode's lock is never freed description: A create whose inode slot stays locked on another node past lock.DefaultTimeout fails with EIO, not ENOSPC, and its directory entry is rolled back