From f97f85203312b387a453047200e97112342d495c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:38:25 -0500 Subject: [PATCH 1/9] fix(trim): never discard a block that was allocated again bitmap.Free cleared the bit and queued the block for TRIM, and the discard ran up to MaxAge (5s) or 256 frees later. Allocation scans from the lowest free block, so any node could allocate the block and write to it first, and BLKDISCARD then destroyed the new data. Image files reject BLKDISCARD, so no test saw it; on a device that supports discard it loses data. The TRIM manager now discards through a guard the bitmap registers in SetTrimManager. The guard takes the bitmap lock, discards only the blocks still free on disk, and holds the lock until the discard ends. Queue no longer flushes inline: Free holds the bitmap lock, so a full batch wakes the flush loop instead. Duplicate queued blocks (freed, reallocated, freed again) are discarded once. Shared blocks reach Free only at refcount 0, so they are covered by the same check. Pending discards live only in memory: a crash before the flush leaves those blocks free and undiscarded, which costs space on thin storage but no data. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 47 ++++++++++ internal/bitmap/trim_test.go | 161 +++++++++++++++++++++++++++++++++++ internal/trim/trim.go | 155 ++++++++++++++++++++------------- internal/trim/trim_test.go | 61 ++++++------- 4 files changed, 333 insertions(+), 91 deletions(-) create mode 100644 internal/bitmap/trim_test.go diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 55655bf7..8e7e5df6 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -270,10 +270,57 @@ func (r *Region) AllocatedCount() uint64 { // SetTrimManager sets the TRIM manager for batched block discards. // When set, freed blocks will be queued for TRIM rather than immediately discarded. +// A manager that accepts a free guard gets one, so it skips blocks allocated again +// before its discard runs. func (r *Region) SetTrimManager(tm TrimQueuer) { r.mu.Lock() defer r.mu.Unlock() r.trimMgr = tm + if g, ok := tm.(interface { + SetFreeGuard(func(context.Context, []uint64, func([]uint64)) error) + }); ok { + g.SetFreeGuard(r.withFreeBlocks) + } +} + +// withFreeBlocks calls discard with the blocks that are free on disk, holding +// the allocation lock until discard returns so no node can allocate them. +func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard func(free []uint64)) error { + if r.lockMgr != nil { + if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + return fmt.Errorf("failed to acquire bitmap lock: %w", err) + } + defer r.unlock() + } + + r.mu.Lock() + defer r.mu.Unlock() + + sectors := make(map[uint64][]byte) + free := make([]uint64, 0, len(blocks)) + for _, b := range blocks { + if b >= r.totalBlocks { + continue + } + sectorIdx := b / BitsPerSector + data, ok := sectors[sectorIdx] + if !ok { + var err error + if data, err = r.readSector(sectorIdx); err != nil { + return fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) + } + sectors[sectorIdx] = data + } + bit := b % BitsPerSector + if data[bit/8]&(1<<(bit%8)) == 0 { + free = append(free, b) + } + } + + if len(free) > 0 { + discard(free) + } + return nil } // Scan reads the entire bitmap and returns free/allocated counts. diff --git a/internal/bitmap/trim_test.go b/internal/bitmap/trim_test.go new file mode 100644 index 00000000..8428e386 --- /dev/null +++ b/internal/bitmap/trim_test.go @@ -0,0 +1,161 @@ +package bitmap + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/trim" +) + +type discardRecorder struct { + mu sync.Mutex + discarded map[uint64]bool +} + +func (d *discardRecorder) discard(offset, length uint64) error { + d.mu.Lock() + defer d.mu.Unlock() + for b := offset / trim.BlockSize; b < (offset+length)/trim.BlockSize; b++ { + d.discarded[b] = true + } + return nil +} + +func (d *discardRecorder) has(block uint64) bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.discarded[block] +} + +func newTrimmedRegion(t *testing.T, batch int, maxAge time.Duration) (*Region, *trim.Manager, *discardRecorder, func()) { + t.Helper() + dev, cleanup := createTestDevice(t, 10<<20) + rec := &discardRecorder{discarded: make(map[uint64]bool)} + tm := trim.NewManager(dev, trim.Config{BatchSize: batch, MaxAge: maxAge, Enabled: true, Discard: rec.discard}) + region := NewRegion(dev, 0, 1000, nil, [16]byte{1}) + region.SetTrimManager(tm) + tm.Start() + return region, tm, rec, func() { + _ = tm.Stop() + cleanup() + } +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met within 5s") + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestTrimSkipsBlockReallocatedBeforeFlush(t *testing.T) { + region, tm, rec, cleanup := newTrimmedRegion(t, 1000, 50*time.Millisecond) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := region.Free(ctx, blocks[0]); err != nil { + t.Fatal(err) + } + again, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if again[0] != blocks[0] { + t.Fatalf("Alloc after Free got block %d, want the freed block %d", again[0], blocks[0]) + } + + waitFor(t, func() bool { return tm.Stats().PendingBlocks == 0 }) + if rec.has(blocks[0]) { + t.Errorf("block %d was discarded after it was allocated again", blocks[0]) + } +} + +func TestTrimSkipsBlockAnotherNodeAllocated(t *testing.T) { + region, tm, rec, cleanup := newTrimmedRegion(t, 1000, time.Hour) + defer cleanup() + other := NewRegion(region.dev, 0, 1000, nil, [16]byte{2}) + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := region.Free(ctx, blocks[0]); err != nil { + t.Fatal(err) + } + theirs, err := other.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if theirs[0] != blocks[0] { + t.Fatalf("other node got block %d, want the freed block %d", theirs[0], blocks[0]) + } + + if err := tm.Stop(); err != nil { + t.Fatal(err) + } + if rec.has(blocks[0]) { + t.Errorf("block %d was discarded after another node allocated it", blocks[0]) + } +} + +func TestTrimDiscardsFreedBlocks(t *testing.T) { + region, tm, rec, cleanup := newTrimmedRegion(t, 1000, time.Hour) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 3) + if err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if err := region.Free(ctx, b); err != nil { + t.Fatal(err) + } + } + + if err := tm.Stop(); err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if !rec.has(b) { + t.Errorf("free block %d was not discarded", b) + } + } +} + +// TestTrimBatchFlushFromFree checks that a full batch does not flush inside +// Free, which holds the bitmap lock the flush needs. +func TestTrimBatchFlushFromFree(t *testing.T) { + region, _, rec, cleanup := newTrimmedRegion(t, 1, time.Hour) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { done <- region.Free(ctx, blocks[0]) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Free did not return: the batch flush deadlocked on the bitmap lock") + } + + waitFor(t, func() bool { return rec.has(blocks[0]) }) +} diff --git a/internal/trim/trim.go b/internal/trim/trim.go index d7c13eb0..835e5688 100644 --- a/internal/trim/trim.go +++ b/internal/trim/trim.go @@ -11,6 +11,7 @@ package trim import ( + "context" "log" "sort" "sync" @@ -50,8 +51,19 @@ type Config struct { // Enabled controls whether TRIM operations are actually issued. // When false, blocks are still tracked but no ioctl is issued. Enabled bool + + // Discard replaces the BLKDISCARD ioctl when set. Offsets and lengths are in bytes. + Discard func(offset, length uint64) error } +// FreeGuard calls discard with the blocks that are still free, and keeps them +// from being allocated until discard returns. Freed blocks wait in the queue +// for up to MaxAge, and any node may allocate and write one in that time. +type FreeGuard = func(ctx context.Context, blocks []uint64, discard func(free []uint64)) error + +// guardTimeout bounds how long a flush waits for the guard's allocation lock. +const guardTimeout = 30 * time.Second + // DefaultConfig returns the default TRIM configuration. func DefaultConfig() Config { return Config{ @@ -82,9 +94,11 @@ type Manager struct { // Pending blocks for TRIM pending []pendingBlock + guard FreeGuard // Background loop control stopCh chan struct{} + flushCh chan struct{} flushWg sync.WaitGroup stopOnce sync.Once // Ensures Stop() is idempotent @@ -102,9 +116,17 @@ func NewManager(dev *storage.Device, cfg Config) *Manager { config: cfg, pending: make([]pendingBlock, 0, cfg.BatchSize), stopCh: make(chan struct{}), + flushCh: make(chan struct{}, 1), } } +// SetFreeGuard makes every flush discard only the blocks the guard reports as free. +func (m *Manager) SetFreeGuard(guard FreeGuard) { + m.mu.Lock() + defer m.mu.Unlock() + m.guard = guard +} + // Start begins the background flush loop. func (m *Manager) Start() { m.flushWg.Add(1) @@ -120,17 +142,11 @@ func (m *Manager) Stop() error { close(m.stopCh) m.flushWg.Wait() - // Final flush of any remaining blocks - m.mu.Lock() - defer m.mu.Unlock() - - if len(m.pending) > 0 { - log.Printf("[trim] final flush of %d pending blocks", len(m.pending)) - m.flushLocked() - } + m.flush("final") + stats := m.Stats() log.Printf("[trim] stopped (queued=%d, discarded=%d, calls=%d)", - m.blocksQueued, m.blocksDiscarded, m.discardCalls) + stats.BlocksQueued, stats.BlocksDiscarded, stats.DiscardCalls) }) return nil } @@ -146,11 +162,19 @@ func (m *Manager) Queue(blockNum uint64) { queuedAt: time.Now(), }) m.blocksQueued++ + m.wakeIfFullLocked() +} - // Check if batch size threshold reached - if len(m.pending) >= m.config.BatchSize { - log.Printf("[trim] batch threshold reached (%d blocks)", len(m.pending)) - m.flushLocked() +// wakeIfFullLocked asks the flush loop to flush once the batch is full. Queue +// runs inside bitmap.Free, which holds the bitmap lock the guard needs, so it +// must not flush itself. +func (m *Manager) wakeIfFullLocked() { + if len(m.pending) < m.config.BatchSize { + return + } + select { + case m.flushCh <- struct{}{}: + default: } } @@ -168,11 +192,7 @@ func (m *Manager) QueueRange(startBlock, count uint64) { }) m.blocksQueued++ } - - if len(m.pending) >= m.config.BatchSize { - log.Printf("[trim] batch threshold reached (%d blocks)", len(m.pending)) - m.flushLocked() - } + m.wakeIfFullLocked() } // flushLoop runs periodically to check age-based flush conditions. @@ -187,6 +207,8 @@ func (m *Manager) flushLoop() { select { case <-m.stopCh: return + case <-m.flushCh: + m.flush("batch threshold") case <-ticker.C: m.checkAndFlush() } @@ -196,41 +218,56 @@ func (m *Manager) flushLoop() { // checkAndFlush flushes if the oldest pending block exceeds MaxAge. func (m *Manager) checkAndFlush() { m.mu.Lock() - defer m.mu.Unlock() - - if len(m.pending) == 0 { - return - } - - // Find oldest entry - oldest := m.pending[0].queuedAt - for _, p := range m.pending { - if p.queuedAt.Before(oldest) { - oldest = p.queuedAt + due := false + if len(m.pending) > 0 { + oldest := m.pending[0].queuedAt + for _, p := range m.pending { + if p.queuedAt.Before(oldest) { + oldest = p.queuedAt + } } + due = time.Since(oldest) >= m.config.MaxAge } + m.mu.Unlock() - if time.Since(oldest) >= m.config.MaxAge { - log.Printf("[trim] age threshold reached (%d blocks, oldest=%v)", - len(m.pending), time.Since(oldest)) - m.flushLocked() + if due { + m.flush("age threshold") } } -// flushLocked issues BLKDISCARD for all pending blocks. -// Must be called with m.mu held. -func (m *Manager) flushLocked() { - if len(m.pending) == 0 { +// flush discards the pending blocks. It runs the guard without holding m.mu, +// because bitmap.Free holds the bitmap lock while it calls Queue. +func (m *Manager) flush(reason string) { + m.mu.Lock() + blocks := make([]uint64, len(m.pending)) + for i, p := range m.pending { + blocks[i] = p.blockNum + } + m.pending = m.pending[:0] + guard := m.guard + m.mu.Unlock() + + if len(blocks) == 0 { return } + log.Printf("[trim] %s: flushing %d blocks", reason, len(blocks)) - // Coalesce adjacent blocks into ranges - ranges := m.coalesceBlocks() + if guard == nil { + m.discardBlocks(blocks) + return + } + ctx, cancel := context.WithTimeout(context.Background(), guardTimeout) + defer cancel() + if err := guard(ctx, blocks, m.discardBlocks); err != nil { + log.Printf("[trim] skipping discard of %d blocks: %v", len(blocks), err) + } +} - log.Printf("[trim] flushing %d blocks in %d ranges", len(m.pending), len(ranges)) +// discardBlocks issues one discard per contiguous range of blocks. +func (m *Manager) discardBlocks(blocks []uint64) { + ranges := coalesceBlocks(blocks) - // Issue BLKDISCARD for each range - discarded := uint64(0) + var discarded, calls uint64 for _, r := range ranges { if err := m.issueDiscard(r); err != nil { log.Printf("[trim] discard failed for range [%d, %d): %v", @@ -238,40 +275,39 @@ func (m *Manager) flushLocked() { continue } discarded += r.count - m.discardCalls++ + calls++ } + m.mu.Lock() + defer m.mu.Unlock() m.blocksDiscarded += discarded + m.discardCalls += calls if len(ranges) > 0 { m.coalesceRatio = float64(discarded) / float64(len(ranges)) } - - // Clear pending list - m.pending = m.pending[:0] } -// coalesceBlocks groups adjacent blocks into contiguous ranges. -// Returns sorted ranges for efficient BLKDISCARD. -func (m *Manager) coalesceBlocks() []blockRange { - if len(m.pending) == 0 { +// coalesceBlocks sorts blocks, drops duplicates and groups adjacent blocks into +// contiguous ranges. A block freed, reallocated and freed again is queued twice. +func coalesceBlocks(blocks []uint64) []blockRange { + if len(blocks) == 0 { return nil } - // Sort blocks by block number - sort.Slice(m.pending, func(i, j int) bool { - return m.pending[i].blockNum < m.pending[j].blockNum - }) + sort.Slice(blocks, func(i, j int) bool { return blocks[i] < blocks[j] }) var ranges []blockRange current := blockRange{ - startBlock: m.pending[0].blockNum, + startBlock: blocks[0], count: 1, } - for i := 1; i < len(m.pending); i++ { - blockNum := m.pending[i].blockNum + for i := 1; i < len(blocks); i++ { + blockNum := blocks[i] - // Check if adjacent to current range + if blockNum < current.startBlock+current.count { + continue + } if blockNum == current.startBlock+current.count { current.count++ } else { @@ -303,6 +339,9 @@ func (m *Manager) issueDiscard(r blockRange) error { // startBlock is relative to data region, add DataOffset byteOffset := m.config.DataOffset + r.startBlock*BlockSize byteLength := r.count * BlockSize + if m.config.Discard != nil { + return m.config.Discard(byteOffset, byteLength) + } // BLKDISCARD takes [2]uint64{offset, length} args := [2]uint64{byteOffset, byteLength} diff --git a/internal/trim/trim_test.go b/internal/trim/trim_test.go index 16d1dce7..fffdd481 100644 --- a/internal/trim/trim_test.go +++ b/internal/trim/trim_test.go @@ -9,18 +9,7 @@ import ( ) func TestCoalesceBlocks(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 10, queuedAt: time.Now()}, - {blockNum: 11, queuedAt: time.Now()}, - {blockNum: 12, queuedAt: time.Now()}, - {blockNum: 20, queuedAt: time.Now()}, - {blockNum: 21, queuedAt: time.Now()}, - {blockNum: 50, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{10, 11, 12, 20, 21, 50}) if len(ranges) != 3 { t.Fatalf("expected 3 ranges, got %d", len(ranges)) @@ -46,24 +35,14 @@ func TestCoalesceBlocks(t *testing.T) { } func TestCoalesceBlocksEmpty(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{}, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks(nil) if ranges != nil { t.Errorf("expected nil for empty pending, got %v", ranges) } } func TestCoalesceBlocksSingleBlock(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 42, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{42}) if len(ranges) != 1 { t.Fatalf("expected 1 range, got %d", len(ranges)) } @@ -75,15 +54,7 @@ func TestCoalesceBlocksSingleBlock(t *testing.T) { func TestCoalesceBlocksUnsorted(t *testing.T) { // Blocks out of order - should still coalesce correctly - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 12, queuedAt: time.Now()}, - {blockNum: 10, queuedAt: time.Now()}, - {blockNum: 11, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{12, 10, 11}) if len(ranges) != 1 { t.Fatalf("expected 1 range (after sorting), got %d", len(ranges)) } @@ -352,3 +323,27 @@ func TestStatsAreThreadSafe(t *testing.T) { t.Errorf("expected 100 queued, got %d", stats.BlocksQueued) } } + +func TestDuplicateBlockDiscardedOnce(t *testing.T) { + dev, cleanup := createTestDevice(t) + defer cleanup() + + var calls [][2]uint64 + m := NewManager(dev, Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + calls = append(calls, [2]uint64{offset, length}) + return nil + }}) + m.Start() + + m.Queue(5) + m.Queue(5) + m.Queue(6) + if err := m.Stop(); err != nil { + t.Fatal(err) + } + + want := [][2]uint64{{5 * BlockSize, 2 * BlockSize}} + if len(calls) != 1 || calls[0] != want[0] { + t.Errorf("discards = %v, want %v", calls, want) + } +} From 02e88b26a8c1276f0604e6ad1932c909b4878d62 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:41:59 -0500 Subject: [PATCH 2/9] fix(broadcast): deduplicate messages per sender The receiver dropped any message whose sequence number it had seen in the last 5 minutes, from any sender. Each node's sequence starts at its boot time in nanoseconds and counts up by one, so once one node has sent more messages than the gap between two boot times (1e7 for a 10ms gap), its sequence numbers overlap the other's and real invalidations from the other node are dropped as duplicates. Peers then serve stale data until the cache TTL. Dedup now keys on sender UUID and sequence. Co-Authored-By: Claude Opus 5.5 --- internal/cache/broadcast/broadcaster_test.go | 40 +++++++++++++++++--- internal/cache/broadcast/receiver.go | 26 ++++++++----- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/internal/cache/broadcast/broadcaster_test.go b/internal/cache/broadcast/broadcaster_test.go index 5413c748..94621ae5 100644 --- a/internal/cache/broadcast/broadcaster_test.go +++ b/internal/cache/broadcast/broadcaster_test.go @@ -98,6 +98,7 @@ func TestBroadcasterDisabled(t *testing.T) { } func TestBroadcasterDuplication(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -113,27 +114,28 @@ func TestBroadcasterDuplication(t *testing.T) { seq2 := uint64(2000) // First time should not be duplicate - if b.isDuplicate(seq1) { + if b.isDuplicate(peer, seq1) { t.Error("First check of seq1 should not be duplicate") } // Second time should be duplicate - if !b.isDuplicate(seq1) { + if !b.isDuplicate(peer, seq1) { t.Error("Second check of seq1 should be duplicate") } // Different sequence should not be duplicate - if b.isDuplicate(seq2) { + if b.isDuplicate(peer, seq2) { t.Error("First check of seq2 should not be duplicate") } // Original sequence still duplicate - if !b.isDuplicate(seq1) { + if !b.isDuplicate(peer, seq1) { t.Error("Third check of seq1 should still be duplicate") } } func TestBroadcasterCleanupSeen(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -146,7 +148,7 @@ func TestBroadcasterCleanupSeen(t *testing.T) { // Add some entries for i := uint64(1); i <= 10; i++ { - b.isDuplicate(i) + b.isDuplicate(peer, i) } b.seenMu.Lock() @@ -424,6 +426,7 @@ func TestConfigDisabled(t *testing.T) { // TestConcurrentDeduplication tests thread-safety of deduplication. func TestConcurrentDeduplication(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -443,7 +446,7 @@ func TestConcurrentDeduplication(t *testing.T) { go func() { defer wg.Done() for j := 0; j < 100; j++ { - if b.isDuplicate(12345) { + if b.isDuplicate(peer, 12345) { atomic.AddInt64(&duplicates, 1) } } @@ -458,3 +461,28 @@ func TestConcurrentDeduplication(t *testing.T) { t.Errorf("Expected at least 990 duplicates, got %d", duplicates) } } + +// TestSameSequenceFromTwoSenders checks that dedup is per sender. Each node's +// sequence starts at its boot time in nanoseconds, so ranges can overlap. +func TestSameSequenceFromTwoSenders(t *testing.T) { + handler := newMockHandler() + cfg := DefaultConfig() + cfg.Enabled = false + + b, err := NewBroadcaster(cfg, [16]byte{1}, 0, handler) + if err != nil { + t.Fatalf("NewBroadcaster failed: %v", err) + } + + for i, sender := range [][16]byte{{2}, {3}} { + msg := NewMessageBuilder(sender).InvalidateInode(uint32(10+i), 1, 1) + msg.Sequence = 777 + b.handleMessage(msg, nil) + } + + handler.mu.Lock() + defer handler.mu.Unlock() + if len(handler.inodeInvalidated) != 2 { + t.Errorf("invalidated inodes = %v, want both senders' inodes [10 11]", handler.inodeInvalidated) + } +} diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index e3323ee1..71d43f91 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -33,7 +33,7 @@ type Broadcaster struct { // Deduplication seenMu sync.RWMutex - seen map[uint64]time.Time // sequence -> first seen time + seen map[seenKey]time.Time // first seen time // Stats invalidationsReceived uint64 @@ -48,6 +48,13 @@ type Broadcaster struct { running atomic.Bool } +// seenKey identifies a message. Each node's sequence starts at its boot time in +// nanoseconds, so two nodes' sequence ranges can overlap. +type seenKey struct { + sender [16]byte + seq uint64 +} + // NewBroadcaster creates a new cache invalidation broadcaster. func NewBroadcaster(cfg Config, localUUID [16]byte, nodeSlot uint32, handler InvalidationHandler) (*Broadcaster, error) { transport, err := NewTransport(cfg) @@ -62,7 +69,7 @@ func NewBroadcaster(cfg Config, localUUID [16]byte, nodeSlot uint32, handler Inv localUUID: localUUID, nodeSlot: nodeSlot, localPort: uint16(cfg.UnicastPort), - seen: make(map[uint64]time.Time), + seen: make(map[seenKey]time.Time), } return b, nil @@ -132,7 +139,7 @@ func (b *Broadcaster) handleMessage(msg *Message, addr *net.UDPAddr) { } // Deduplicate - if b.isDuplicate(msg.Sequence) { + if b.isDuplicate(msg.SenderUUID, msg.Sequence) { atomic.AddUint64(&b.duplicatesDropped, 1) return } @@ -226,16 +233,17 @@ func (b *Broadcaster) handlePeerAnnounce(msg *Message, addr *net.UDPAddr) { payload.NodeSlot, uuidStr[:16], peerAddr) } -// isDuplicate checks if we've seen this sequence number recently. -func (b *Broadcaster) isDuplicate(seq uint64) bool { +// isDuplicate checks if we've seen this message from this sender recently. +func (b *Broadcaster) isDuplicate(sender [16]byte, seq uint64) bool { b.seenMu.Lock() defer b.seenMu.Unlock() - if _, exists := b.seen[seq]; exists { + key := seenKey{sender: sender, seq: seq} + if _, exists := b.seen[key]; exists { return true } - b.seen[seq] = time.Now() + b.seen[key] = time.Now() return false } @@ -281,9 +289,9 @@ func (b *Broadcaster) cleanupSeen() { defer b.seenMu.Unlock() cutoff := time.Now().Add(-5 * time.Minute) - for seq, seen := range b.seen { + for key, seen := range b.seen { if seen.Before(cutoff) { - delete(b.seen, seq) + delete(b.seen, key) } } } From fb6250333fd5f4bc85da4ca14a9341e92b8b5a5c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:41:59 -0500 Subject: [PATCH 3/9] fix(broadcast): reject bytes after the HMAC and test message integrity Unmarshal accepted datagrams with extra bytes after the payload and HMAC. They were ignored, but they were not authenticated. It now rejects them, so an accepted datagram is authenticated in full. New tests flip every bit position of a signed message, cut it at every length, strip its HMAC, and check that the largest message fits in one datagram on a 1500-byte MTU. Every invalidation carries one inode, so messages are 62 bytes (94 with HMAC) and never need IP fragmentation. FuzzUnmarshal checks that any accepted signed datagram is exactly the bytes Marshal produces. Co-Authored-By: Claude Opus 5.5 --- internal/cache/broadcast/integrity_test.go | 111 +++++++++++++++++++++ internal/cache/broadcast/protocol.go | 4 + 2 files changed, 115 insertions(+) create mode 100644 internal/cache/broadcast/integrity_test.go diff --git a/internal/cache/broadcast/integrity_test.go b/internal/cache/broadcast/integrity_test.go new file mode 100644 index 00000000..2b8a55c7 --- /dev/null +++ b/internal/cache/broadcast/integrity_test.go @@ -0,0 +1,111 @@ +package broadcast + +import ( + "bytes" + "testing" +) + +func signedInvalidation(t *testing.T, secret []byte) []byte { + t.Helper() + msg := NewMessageBuilder([16]byte{0xAB}).InvalidateInode(42, 7, 9) + data, err := msg.Marshal(secret) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestHMACCoversEveryByte(t *testing.T) { + secret := []byte("cluster-secret") + data := signedInvalidation(t, secret) + + for i := range data { + for _, bit := range []byte{0x01, 0x80} { + tampered := bytes.Clone(data) + tampered[i] ^= bit + if _, err := Unmarshal(tampered, secret); err == nil { + t.Errorf("flipping bit %#x of byte %d was accepted", bit, i) + } + } + } + + if _, err := Unmarshal(append(bytes.Clone(data), 0), secret); err == nil { + t.Error("a trailing byte outside the HMAC was accepted") + } + + stripped := bytes.Clone(data[:len(data)-HMACSize]) + stripped[6] &^= FlagHMAC + if _, err := Unmarshal(stripped, secret); err == nil { + t.Error("a message with its HMAC stripped was accepted") + } +} + +func TestTruncatedMessageRejected(t *testing.T) { + for _, secret := range [][]byte{nil, []byte("cluster-secret")} { + data := signedInvalidation(t, secret) + for n := 0; n < len(data); n++ { + if msg, err := Unmarshal(data[:n], secret); err == nil { + t.Errorf("secret=%t: %d of %d bytes parsed as %+v", secret != nil, n, len(data), msg) + } + } + } +} + +// TestLargestMessageFitsOneDatagram checks that no message needs IP +// fragmentation on a 1500-byte MTU, where one lost fragment drops the message. +func TestLargestMessageFitsOneDatagram(t *testing.T) { + const ipv6UDPPayload = 1500 - 40 - 8 + if MaxMessageSize > ipv6UDPPayload { + t.Fatalf("MaxMessageSize %d exceeds the %d-byte UDP payload of a 1500-byte IPv6 MTU", MaxMessageSize, ipv6UDPPayload) + } + + largest := &Message{Version: ProtocolVersion, Type: TypeInvalidateInode, Payload: make([]byte, MaxPayloadSize)} + data, err := largest.Marshal([]byte("cluster-secret")) + if err != nil { + t.Fatalf("largest allowed message failed to marshal: %v", err) + } + if len(data) > MaxMessageSize { + t.Errorf("largest message is %d bytes, over MaxMessageSize %d", len(data), MaxMessageSize) + } + + tooBig := &Message{Version: ProtocolVersion, Type: TypeInvalidateInode, Payload: make([]byte, MaxPayloadSize+1)} + if _, err := tooBig.Marshal(nil); err == nil { + t.Error("payload over MaxPayloadSize was accepted") + } +} + +func FuzzUnmarshal(f *testing.F) { + secret := []byte("cluster-secret") + builder := NewMessageBuilder([16]byte{1}) + for _, msg := range []*Message{ + builder.InvalidateInode(1, 2, 3), + builder.InvalidateExtent(4, 5), + builder.InvalidateAll(), + builder.Heartbeat(), + builder.PeerAnnounce(1, 31337, [16]byte{}), + } { + for _, s := range [][]byte{nil, secret} { + data, err := msg.Marshal(s) + if err != nil { + f.Fatal(err) + } + f.Add(data) + } + } + + f.Fuzz(func(t *testing.T, data []byte) { + for _, s := range [][]byte{nil, secret} { + msg, err := Unmarshal(data, s) + if err != nil { + continue + } + again, err := msg.Marshal(s) + if err != nil { + t.Fatalf("accepted message does not marshal: %v", err) + } + if s != nil && !bytes.Equal(again, data) { + t.Fatalf("accepted bytes differ from the authenticated message:\n got %x\nwant %x", data, again) + } + } + }) +} diff --git a/internal/cache/broadcast/protocol.go b/internal/cache/broadcast/protocol.go index 377c5075..f967330f 100644 --- a/internal/cache/broadcast/protocol.go +++ b/internal/cache/broadcast/protocol.go @@ -209,6 +209,10 @@ func Unmarshal(data []byte, secret []byte) (*Message, error) { if len(data) < expectedSize { return nil, ErrMessageTooShort } + if len(data) > expectedSize { + // Trailing bytes would not be covered by the HMAC + return nil, ErrMessageTooLong + } // Verify HMAC if present and secret provided if hasHMAC { From 49b1c602c0b368bd3e80724477f53bd4d098b0f8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 17:42:54 -0500 Subject: [PATCH 4/9] test: list the TRIM and broadcast safety tests in the manifest Co-Authored-By: Claude Opus 5.5 --- tests/test-manifest.yaml | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index df005977..a72a29c7 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1091,6 +1091,41 @@ scenarios: environments: go: internal/trim/trim_test.go::TestStatsAreThreadSafe + - id: trim-skip-reallocated + name: TRIM skips a reallocated block + description: A block freed and allocated again before the flush is not discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimSkipsBlockReallocatedBeforeFlush + + - id: trim-skip-remote-allocated + name: TRIM skips a block another node allocated + description: A freed block another node allocated before the flush is not discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimSkipsBlockAnotherNodeAllocated + + - id: trim-discard-free + name: TRIM discards free blocks + description: Blocks still free at flush time are discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimDiscardsFreedBlocks + + - id: trim-batch-flush-from-free + name: TRIM batch flush from Free + description: A full batch flushes in the background without deadlocking Free on the bitmap lock + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimBatchFlushFromFree + + - id: trim-duplicate-block + name: TRIM duplicate block + description: A block queued twice is discarded once + priority: go_only + environments: + go: internal/trim/trim_test.go::TestDuplicateBlockDiscardedOnce + # =========================================================================== # SPACE REPORTING TESTS (Phase 5: Thin Provisioning) # =========================================================================== @@ -1458,6 +1493,41 @@ scenarios: environments: go: internal/cache/broadcast/broadcaster_test.go::TestConfigDefaults + - id: broadcast-dedup-per-sender + name: Broadcast dedup per sender + description: The same sequence number from two senders is processed twice + priority: go_only + environments: + go: internal/cache/broadcast/broadcaster_test.go::TestSameSequenceFromTwoSenders + + - id: broadcast-hmac-every-byte + name: Broadcast HMAC covers every byte + description: Flipping any bit, appending a byte or stripping the HMAC is rejected + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestHMACCoversEveryByte + + - id: broadcast-truncation + name: Broadcast truncated messages + description: Every truncation of a message is rejected + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestTruncatedMessageRejected + + - id: broadcast-single-datagram + name: Broadcast message fits one datagram + description: The largest message fits a 1500-byte MTU without fragmentation + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestLargestMessageFitsOneDatagram + + - id: broadcast-fuzz-unmarshal + name: Broadcast Unmarshal fuzz + description: Accepted signed datagrams are exactly what Marshal produces + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::FuzzUnmarshal + # --------------------------------------------------------------------------- # Broadcast Integration Tests (end-to-end cross-node invalidation) # --------------------------------------------------------------------------- From 4fd15fcedd87b2a28b023e7c0222f5c9515a6b92 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:20:00 -0500 Subject: [PATCH 5/9] fix(trim): reserve blocks during a discard instead of holding the bitmap lock The guard held the bitmap lock for the whole discard. Nothing renews that lock and peers take it over once it is 15s old, so a discard slower than about 9-15s (thin LVM, SAN UNMAP, zeroing devices) let another node allocate and write a block that was still being discarded. It also stalled every node's allocations meanwhile. The guard now marks the still-free blocks allocated under the lock, releases it, discards, then frees exactly those blocks under the lock. A crash during the discard leaves them allocated and unreferenced, which fsck reports as leaked. SetFreeGuard is now part of TrimQueuer, so a TRIM manager that does not accept the guard no longer compiles. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 87 ++++++++++++++++++++++++++------------- internal/trim/trim.go | 16 ++++--- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 8e7e5df6..6d504d89 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -50,6 +50,8 @@ type TrimQueuer interface { Queue(blockNum uint64) // QueueRange adds multiple contiguous freed blocks to the TRIM queue. QueueRange(startBlock, count uint64) + // SetFreeGuard makes the manager discard only blocks the guard confirms are free. + SetFreeGuard(guard func(ctx context.Context, blocks []uint64, discard func(free []uint64)) error) } // Region manages block allocation using a bitmap stored on disk. @@ -270,25 +272,47 @@ func (r *Region) AllocatedCount() uint64 { // SetTrimManager sets the TRIM manager for batched block discards. // When set, freed blocks will be queued for TRIM rather than immediately discarded. -// A manager that accepts a free guard gets one, so it skips blocks allocated again -// before its discard runs. func (r *Region) SetTrimManager(tm TrimQueuer) { r.mu.Lock() defer r.mu.Unlock() r.trimMgr = tm - if g, ok := tm.(interface { - SetFreeGuard(func(context.Context, []uint64, func([]uint64)) error) - }); ok { - g.SetFreeGuard(r.withFreeBlocks) + if tm != nil { + tm.SetFreeGuard(r.withFreeBlocks) } } -// withFreeBlocks calls discard with the blocks that are free on disk, holding -// the allocation lock until discard returns so no node can allocate them. +// withFreeBlocks marks the blocks that are still free as allocated, calls +// discard without the bitmap lock, then frees them again. A discard can outlast +// the lock's stale threshold, so no lock is held while it runs. A crash during +// the discard leaves the blocks allocated and unreferenced, which fsck reports. func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard func(free []uint64)) error { + reserved, err := r.flipBits(ctx, blocks, true) + if err != nil { + if len(reserved) > 0 { + if _, rerr := r.flipBits(ctx, reserved, false); rerr != nil { + err = errors.Join(err, fmt.Errorf("blocks %v stay allocated: %w", reserved, rerr)) + } + } + return err + } + if len(reserved) == 0 { + return nil + } + + discard(reserved) + + if _, err := r.flipBits(ctx, reserved, false); err != nil { + return fmt.Errorf("blocks %v stay allocated after discard: %w", reserved, err) + } + return nil +} + +// flipBits marks each block allocated (or free) whose bit says otherwise, under +// the bitmap lock, and returns the blocks it changed, even on error. +func (r *Region) flipBits(ctx context.Context, blocks []uint64, allocated bool) ([]uint64, error) { if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { - return fmt.Errorf("failed to acquire bitmap lock: %w", err) + return nil, fmt.Errorf("failed to acquire bitmap lock: %w", err) } defer r.unlock() } @@ -296,31 +320,38 @@ func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard fu r.mu.Lock() defer r.mu.Unlock() - sectors := make(map[uint64][]byte) - free := make([]uint64, 0, len(blocks)) + bySector := make(map[uint64][]uint64) for _, b := range blocks { - if b >= r.totalBlocks { - continue + if b < r.totalBlocks { + bySector[b/BitsPerSector] = append(bySector[b/BitsPerSector], b) + } + } + + var changed []uint64 + for sectorIdx, sectorBlocks := range bySector { + data, err := r.readSector(sectorIdx) + if err != nil { + return changed, fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) } - sectorIdx := b / BitsPerSector - data, ok := sectors[sectorIdx] - if !ok { - var err error - if data, err = r.readSector(sectorIdx); err != nil { - return fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) + var flipped []uint64 + for _, b := range sectorBlocks { + bit := b % BitsPerSector + mask := byte(1) << (bit % 8) + if (data[bit/8]&mask != 0) == allocated { + continue } - sectors[sectorIdx] = data + data[bit/8] ^= mask + flipped = append(flipped, b) } - bit := b % BitsPerSector - if data[bit/8]&(1<<(bit%8)) == 0 { - free = append(free, b) + if len(flipped) == 0 { + continue } + if err := r.writeSector(sectorIdx, data); err != nil { + return changed, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + changed = append(changed, flipped...) } - - if len(free) > 0 { - discard(free) - } - return nil + return changed, nil } // Scan reads the entire bitmap and returns free/allocated counts. diff --git a/internal/trim/trim.go b/internal/trim/trim.go index 835e5688..75e1c43f 100644 --- a/internal/trim/trim.go +++ b/internal/trim/trim.go @@ -56,9 +56,8 @@ type Config struct { Discard func(offset, length uint64) error } -// FreeGuard calls discard with the blocks that are still free, and keeps them -// from being allocated until discard returns. Freed blocks wait in the queue -// for up to MaxAge, and any node may allocate and write one in that time. +// FreeGuard calls discard with the queued blocks that are still free and keeps +// them from being allocated until discard returns. type FreeGuard = func(ctx context.Context, blocks []uint64, discard func(free []uint64)) error // guardTimeout bounds how long a flush waits for the guard's allocation lock. @@ -165,9 +164,8 @@ func (m *Manager) Queue(blockNum uint64) { m.wakeIfFullLocked() } -// wakeIfFullLocked asks the flush loop to flush once the batch is full. Queue -// runs inside bitmap.Free, which holds the bitmap lock the guard needs, so it -// must not flush itself. +// wakeIfFullLocked wakes the flush loop. Queue runs inside bitmap.Free, which +// holds the bitmap lock the guard needs, so it must not flush itself. func (m *Manager) wakeIfFullLocked() { if len(m.pending) < m.config.BatchSize { return @@ -235,8 +233,8 @@ func (m *Manager) checkAndFlush() { } } -// flush discards the pending blocks. It runs the guard without holding m.mu, -// because bitmap.Free holds the bitmap lock while it calls Queue. +// flush discards the pending blocks. The guard runs without m.mu held, because +// Free takes the bitmap lock before m.mu. func (m *Manager) flush(reason string) { m.mu.Lock() blocks := make([]uint64, len(m.pending)) @@ -288,7 +286,7 @@ func (m *Manager) discardBlocks(blocks []uint64) { } // coalesceBlocks sorts blocks, drops duplicates and groups adjacent blocks into -// contiguous ranges. A block freed, reallocated and freed again is queued twice. +// contiguous ranges. func coalesceBlocks(blocks []uint64) []blockRange { if len(blocks) == 0 { return nil From ad0b67c3385ecfb77b19194a2adfe6bb88ce3b2a Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:20:00 -0500 Subject: [PATCH 6/9] test(trim): make the reallocation test deterministic and run the guard with real locks TestTrimSkipsBlockReallocatedBeforeFlush waited for the queue to empty, which happens before the discard runs, so it passed 13 of 30 runs with the guard removed. It now also frees a higher block and waits for that block's discard. It fails 30 of 30 runs without the guard. TestTrimDiscardHoldsNoBitmapLock uses two real lock managers. While node A discards, the block must read as allocated and node B must still allocate a different block within 5s. It fails with the earlier guard that held the lock during the discard. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/trim_test.go | 81 ++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 9 deletions(-) diff --git a/internal/bitmap/trim_test.go b/internal/bitmap/trim_test.go index 8428e386..7e8f31d1 100644 --- a/internal/bitmap/trim_test.go +++ b/internal/bitmap/trim_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/trim" ) @@ -55,28 +56,33 @@ func waitFor(t *testing.T, cond func() bool) { } func TestTrimSkipsBlockReallocatedBeforeFlush(t *testing.T) { - region, tm, rec, cleanup := newTrimmedRegion(t, 1000, 50*time.Millisecond) + region, _, rec, cleanup := newTrimmedRegion(t, 1000, 50*time.Millisecond) defer cleanup() ctx := context.Background() - blocks, err := region.Alloc(ctx, 1) + blocks, err := region.Alloc(ctx, 3) if err != nil { t.Fatal(err) } - if err := region.Free(ctx, blocks[0]); err != nil { - t.Fatal(err) + reused, stillFree := blocks[0], blocks[2] + for _, b := range []uint64{reused, stillFree} { + if err := region.Free(ctx, b); err != nil { + t.Fatal(err) + } } again, err := region.Alloc(ctx, 1) if err != nil { t.Fatal(err) } - if again[0] != blocks[0] { - t.Fatalf("Alloc after Free got block %d, want the freed block %d", again[0], blocks[0]) + if again[0] != reused { + t.Fatalf("Alloc after Free got block %d, want the freed block %d", again[0], reused) } - waitFor(t, func() bool { return tm.Stats().PendingBlocks == 0 }) - if rec.has(blocks[0]) { - t.Errorf("block %d was discarded after it was allocated again", blocks[0]) + // Ranges are discarded in block order, so once stillFree is discarded the + // flush is past reused. + waitFor(t, func() bool { return rec.has(stillFree) }) + if rec.has(reused) { + t.Errorf("block %d was discarded after it was allocated again", reused) } } @@ -159,3 +165,60 @@ func TestTrimBatchFlushFromFree(t *testing.T) { waitFor(t, func() bool { return rec.has(blocks[0]) }) } + +// TestTrimDiscardHoldsNoBitmapLock runs the guard with real lock managers on two +// nodes. A discard can outlast the lock's 15s stale threshold, so the block must +// stay reserved without the lock held, and other nodes must keep allocating. +func TestTrimDiscardHoldsNoBitmapLock(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + lockA := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{1}, 0) + lockB := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{2}, 1) + + started := make(chan struct{}) + finish := make(chan struct{}) + var once sync.Once + tm := trim.NewManager(dev, trim.Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + once.Do(func() { close(started) }) + <-finish + return nil + }}) + a := NewRegion(dev, 0, 1000, lockA, [16]byte{1}) + a.SetTrimManager(tm) + tm.Start() + b := NewRegion(dev, 0, 1000, lockB, [16]byte{2}) + ctx := context.Background() + + blocks, err := a.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := a.Free(ctx, blocks[0]); err != nil { + t.Fatal(err) + } + + stopped := make(chan error, 1) + go func() { stopped <- tm.Stop() }() + <-started + + if free, err := b.IsFree(blocks[0]); err != nil || free { + t.Errorf("block %d being discarded: IsFree = %v, %v; want reserved", blocks[0], free, err) + } + allocCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + theirs, err := b.Alloc(allocCtx, 1) + cancel() + if err != nil { + t.Fatalf("node B could not allocate during node A's discard: %v", err) + } + if theirs[0] == blocks[0] { + t.Errorf("node B allocated block %d while node A was discarding it", blocks[0]) + } + + close(finish) + if err := <-stopped; err != nil { + t.Fatal(err) + } + if free, err := a.IsFree(blocks[0]); err != nil || !free { + t.Errorf("block %d after discard: IsFree = %v, %v; want free again", blocks[0], free, err) + } +} From bba3857f9eafe41f9a73ee2758754868761f064c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:20:14 -0500 Subject: [PATCH 7/9] test: list the two-node TRIM lock test in the manifest Co-Authored-By: Claude Opus 5.5 --- tests/test-manifest.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index a72a29c7..bac485bd 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1119,6 +1119,13 @@ scenarios: environments: go: internal/bitmap/trim_test.go::TestTrimBatchFlushFromFree + - id: trim-discard-without-lock + name: TRIM discard without the bitmap lock + description: With real locks on two nodes, a block being discarded stays reserved and the other node keeps allocating + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimDiscardHoldsNoBitmapLock + - id: trim-duplicate-block name: TRIM duplicate block description: A block queued twice is discarded once From e18d8fe17263790dfc05bd74437f2689c6904535 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:31:44 -0500 Subject: [PATCH 8/9] fix(trim): keep retrying to free reserved blocks after a slow discard The flush passed one 30s context to the guard, and the final free of the reserved blocks used it too. After a discard longer than 30s that context was done, so the free got only the lock manager's minimum retries (about 100ms). If another node held the bitmap lock then, the blocks stayed allocated for good; fsck reports them but cannot repair them. The free now ignores the caller's deadline, takes up to 30s per attempt, and retries every second with a WARN until every reserved block is free. Each retry covers only the blocks still reserved, so a block freed in an earlier round and allocated by another node is not touched again. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 38 ++++++++++++++++++++++----------- internal/bitmap/trim_test.go | 41 ++++++++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 7 ++++++ 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 6d504d89..e608e4b5 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -12,7 +12,9 @@ import ( "fmt" "log" "math/bits" + "slices" "sync" + "time" "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/storage" @@ -288,23 +290,35 @@ func (r *Region) SetTrimManager(tm TrimQueuer) { func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard func(free []uint64)) error { reserved, err := r.flipBits(ctx, blocks, true) if err != nil { - if len(reserved) > 0 { - if _, rerr := r.flipBits(ctx, reserved, false); rerr != nil { - err = errors.Join(err, fmt.Errorf("blocks %v stay allocated: %w", reserved, rerr)) - } - } + r.releaseReserved(ctx, reserved) return err } - if len(reserved) == 0 { - return nil + if len(reserved) > 0 { + discard(reserved) + r.releaseReserved(ctx, reserved) } + return nil +} - discard(reserved) - - if _, err := r.flipBits(ctx, reserved, false); err != nil { - return fmt.Errorf("blocks %v stay allocated after discard: %w", reserved, err) +// releaseTimeout bounds each attempt to take the bitmap lock when freeing reserved blocks. +const releaseTimeout = 30 * time.Second + +// releaseReserved frees blocks reserved for a discard. ctx may have run out +// during a slow discard, and fsck cannot repair a leaked reservation, so it +// retries until every block is free. Nothing else references these blocks. +func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { + ctx = context.WithoutCancel(ctx) + for attempt := 1; len(blocks) > 0; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, releaseTimeout) + freed, err := r.flipBits(attemptCtx, blocks, false) + cancel() + blocks = slices.DeleteFunc(blocks, func(b uint64) bool { return slices.Contains(freed, b) }) + if err == nil { + return + } + log.Printf("[WARN] bitmap: freeing %d blocks reserved for TRIM failed (attempt %d), retrying: %v", len(blocks), attempt, err) + time.Sleep(time.Second) } - return nil } // flipBits marks each block allocated (or free) whose bit says otherwise, under diff --git a/internal/bitmap/trim_test.go b/internal/bitmap/trim_test.go index 7e8f31d1..5a1091ab 100644 --- a/internal/bitmap/trim_test.go +++ b/internal/bitmap/trim_test.go @@ -222,3 +222,44 @@ func TestTrimDiscardHoldsNoBitmapLock(t *testing.T) { t.Errorf("block %d after discard: IsFree = %v, %v; want free again", blocks[0], free, err) } } + +// TestTrimReleasesReservationAfterSlowDiscard checks that the reserved blocks are +// freed even when the flush's context ran out during the discard and another +// node holds the bitmap lock at that moment. +func TestTrimReleasesReservationAfterSlowDiscard(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + lockA := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{1}, 0) + lockB := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{2}, 1) + a := NewRegion(dev, 0, 1000, lockA, [16]byte{1}) + bg := context.Background() + + blocks, err := a.Alloc(bg, 1) + if err != nil { + t.Fatal(err) + } + if err := a.Free(bg, blocks[0]); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(bg, 100*time.Millisecond) + defer cancel() + err = a.withFreeBlocks(ctx, blocks, func([]uint64) { + if err := lockB.Acquire(bg, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + t.Errorf("node B lock: %v", err) + return + } + <-ctx.Done() + go func() { + time.Sleep(time.Second) + _ = lockB.Release(lock.LockTypeBlockBitmap, 0) + }() + }) + if err != nil { + t.Fatal(err) + } + + if free, err := a.IsFree(blocks[0]); err != nil || !free { + t.Errorf("block %d after a discard that outlasted the context: IsFree = %v, %v; want free", blocks[0], free, err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index bac485bd..28803bfd 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1126,6 +1126,13 @@ scenarios: environments: go: internal/bitmap/trim_test.go::TestTrimDiscardHoldsNoBitmapLock + - id: trim-release-after-slow-discard + name: TRIM frees reserved blocks after a slow discard + description: Reserved blocks are freed even when the flush context ran out during the discard and another node holds the bitmap lock + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimReleasesReservationAfterSlowDiscard + - id: trim-duplicate-block name: TRIM duplicate block description: A block queued twice is discarded once From e5ef7099cd7a5566212d0f92651d4a25cbb8f110 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 18:39:42 -0500 Subject: [PATCH 9/9] fix(trim): give up freeing reserved blocks after 5 minutes releaseReserved retried forever. With a persistent device error the TRIM worker spun and unmount's final flush never returned. It now stops after 5 minutes and logs an ERROR naming the blocks still reserved, which fsck will report as leaked. Lock contention clears well within that time. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index e608e4b5..3aa12e67 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -300,15 +300,25 @@ func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard fu return nil } -// releaseTimeout bounds each attempt to take the bitmap lock when freeing reserved blocks. -const releaseTimeout = 30 * time.Second +const ( + // releaseTimeout bounds each attempt to take the bitmap lock when freeing reserved blocks. + releaseTimeout = 30 * time.Second + // releaseDeadline bounds all attempts, so a failing device cannot hang unmount. + releaseDeadline = 5 * time.Minute +) // releaseReserved frees blocks reserved for a discard. ctx may have run out // during a slow discard, and fsck cannot repair a leaked reservation, so it -// retries until every block is free. Nothing else references these blocks. +// retries until every block is free or releaseDeadline passes. Nothing else +// references these blocks. func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { ctx = context.WithoutCancel(ctx) + deadline := time.Now().Add(releaseDeadline) for attempt := 1; len(blocks) > 0; attempt++ { + if time.Now().After(deadline) { + log.Printf("[ERROR] bitmap: gave up freeing blocks %v reserved for TRIM after %v; fsck will report them as leaked", blocks, releaseDeadline) + return + } attemptCtx, cancel := context.WithTimeout(ctx, releaseTimeout) freed, err := r.flipBits(attemptCtx, blocks, false) cancel()