diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 55655bf7..3aa12e67 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -12,7 +12,9 @@ import ( "fmt" "log" "math/bits" + "slices" "sync" + "time" "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/storage" @@ -50,6 +52,8 @@ type TrimQueuer interface { Queue(blockNum uint64) // QueueRange adds multiple contiguous freed blocks to the TRIM queue. QueueRange(startBlock, count uint64) + // SetFreeGuard makes the manager discard only blocks the guard confirms are free. + SetFreeGuard(guard func(ctx context.Context, blocks []uint64, discard func(free []uint64)) error) } // Region manages block allocation using a bitmap stored on disk. @@ -274,6 +278,104 @@ func (r *Region) SetTrimManager(tm TrimQueuer) { r.mu.Lock() defer r.mu.Unlock() r.trimMgr = tm + if tm != nil { + tm.SetFreeGuard(r.withFreeBlocks) + } +} + +// withFreeBlocks marks the blocks that are still free as allocated, calls +// discard without the bitmap lock, then frees them again. A discard can outlast +// the lock's stale threshold, so no lock is held while it runs. A crash during +// the discard leaves the blocks allocated and unreferenced, which fsck reports. +func (r *Region) withFreeBlocks(ctx context.Context, blocks []uint64, discard func(free []uint64)) error { + reserved, err := r.flipBits(ctx, blocks, true) + if err != nil { + r.releaseReserved(ctx, reserved) + return err + } + if len(reserved) > 0 { + discard(reserved) + r.releaseReserved(ctx, reserved) + } + return nil +} + +const ( + // releaseTimeout bounds each attempt to take the bitmap lock when freeing reserved blocks. + releaseTimeout = 30 * time.Second + // releaseDeadline bounds all attempts, so a failing device cannot hang unmount. + releaseDeadline = 5 * time.Minute +) + +// releaseReserved frees blocks reserved for a discard. ctx may have run out +// during a slow discard, and fsck cannot repair a leaked reservation, so it +// retries until every block is free or releaseDeadline passes. Nothing else +// references these blocks. +func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { + ctx = context.WithoutCancel(ctx) + deadline := time.Now().Add(releaseDeadline) + for attempt := 1; len(blocks) > 0; attempt++ { + if time.Now().After(deadline) { + log.Printf("[ERROR] bitmap: gave up freeing blocks %v reserved for TRIM after %v; fsck will report them as leaked", blocks, releaseDeadline) + return + } + attemptCtx, cancel := context.WithTimeout(ctx, releaseTimeout) + freed, err := r.flipBits(attemptCtx, blocks, false) + cancel() + blocks = slices.DeleteFunc(blocks, func(b uint64) bool { return slices.Contains(freed, b) }) + if err == nil { + return + } + log.Printf("[WARN] bitmap: freeing %d blocks reserved for TRIM failed (attempt %d), retrying: %v", len(blocks), attempt, err) + time.Sleep(time.Second) + } +} + +// flipBits marks each block allocated (or free) whose bit says otherwise, under +// the bitmap lock, and returns the blocks it changed, even on error. +func (r *Region) flipBits(ctx context.Context, blocks []uint64, allocated bool) ([]uint64, error) { + if r.lockMgr != nil { + if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + return nil, fmt.Errorf("failed to acquire bitmap lock: %w", err) + } + defer r.unlock() + } + + r.mu.Lock() + defer r.mu.Unlock() + + bySector := make(map[uint64][]uint64) + for _, b := range blocks { + if b < r.totalBlocks { + bySector[b/BitsPerSector] = append(bySector[b/BitsPerSector], b) + } + } + + var changed []uint64 + for sectorIdx, sectorBlocks := range bySector { + data, err := r.readSector(sectorIdx) + if err != nil { + return changed, fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) + } + var flipped []uint64 + for _, b := range sectorBlocks { + bit := b % BitsPerSector + mask := byte(1) << (bit % 8) + if (data[bit/8]&mask != 0) == allocated { + continue + } + data[bit/8] ^= mask + flipped = append(flipped, b) + } + if len(flipped) == 0 { + continue + } + if err := r.writeSector(sectorIdx, data); err != nil { + return changed, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + changed = append(changed, flipped...) + } + return changed, nil } // Scan reads the entire bitmap and returns free/allocated counts. diff --git a/internal/bitmap/trim_test.go b/internal/bitmap/trim_test.go new file mode 100644 index 00000000..5a1091ab --- /dev/null +++ b/internal/bitmap/trim_test.go @@ -0,0 +1,265 @@ +package bitmap + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/trim" +) + +type discardRecorder struct { + mu sync.Mutex + discarded map[uint64]bool +} + +func (d *discardRecorder) discard(offset, length uint64) error { + d.mu.Lock() + defer d.mu.Unlock() + for b := offset / trim.BlockSize; b < (offset+length)/trim.BlockSize; b++ { + d.discarded[b] = true + } + return nil +} + +func (d *discardRecorder) has(block uint64) bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.discarded[block] +} + +func newTrimmedRegion(t *testing.T, batch int, maxAge time.Duration) (*Region, *trim.Manager, *discardRecorder, func()) { + t.Helper() + dev, cleanup := createTestDevice(t, 10<<20) + rec := &discardRecorder{discarded: make(map[uint64]bool)} + tm := trim.NewManager(dev, trim.Config{BatchSize: batch, MaxAge: maxAge, Enabled: true, Discard: rec.discard}) + region := NewRegion(dev, 0, 1000, nil, [16]byte{1}) + region.SetTrimManager(tm) + tm.Start() + return region, tm, rec, func() { + _ = tm.Stop() + cleanup() + } +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met within 5s") + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestTrimSkipsBlockReallocatedBeforeFlush(t *testing.T) { + region, _, rec, cleanup := newTrimmedRegion(t, 1000, 50*time.Millisecond) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 3) + if err != nil { + t.Fatal(err) + } + reused, stillFree := blocks[0], blocks[2] + for _, b := range []uint64{reused, stillFree} { + if err := region.Free(ctx, b); err != nil { + t.Fatal(err) + } + } + again, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if again[0] != reused { + t.Fatalf("Alloc after Free got block %d, want the freed block %d", again[0], reused) + } + + // Ranges are discarded in block order, so once stillFree is discarded the + // flush is past reused. + waitFor(t, func() bool { return rec.has(stillFree) }) + if rec.has(reused) { + t.Errorf("block %d was discarded after it was allocated again", reused) + } +} + +func TestTrimSkipsBlockAnotherNodeAllocated(t *testing.T) { + region, tm, rec, cleanup := newTrimmedRegion(t, 1000, time.Hour) + defer cleanup() + other := NewRegion(region.dev, 0, 1000, nil, [16]byte{2}) + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := region.Free(ctx, blocks[0]); err != nil { + t.Fatal(err) + } + theirs, err := other.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if theirs[0] != blocks[0] { + t.Fatalf("other node got block %d, want the freed block %d", theirs[0], blocks[0]) + } + + if err := tm.Stop(); err != nil { + t.Fatal(err) + } + if rec.has(blocks[0]) { + t.Errorf("block %d was discarded after another node allocated it", blocks[0]) + } +} + +func TestTrimDiscardsFreedBlocks(t *testing.T) { + region, tm, rec, cleanup := newTrimmedRegion(t, 1000, time.Hour) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 3) + if err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if err := region.Free(ctx, b); err != nil { + t.Fatal(err) + } + } + + if err := tm.Stop(); err != nil { + t.Fatal(err) + } + for _, b := range blocks { + if !rec.has(b) { + t.Errorf("free block %d was not discarded", b) + } + } +} + +// TestTrimBatchFlushFromFree checks that a full batch does not flush inside +// Free, which holds the bitmap lock the flush needs. +func TestTrimBatchFlushFromFree(t *testing.T) { + region, _, rec, cleanup := newTrimmedRegion(t, 1, time.Hour) + defer cleanup() + ctx := context.Background() + + blocks, err := region.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { done <- region.Free(ctx, blocks[0]) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Free did not return: the batch flush deadlocked on the bitmap lock") + } + + waitFor(t, func() bool { return rec.has(blocks[0]) }) +} + +// TestTrimDiscardHoldsNoBitmapLock runs the guard with real lock managers on two +// nodes. A discard can outlast the lock's 15s stale threshold, so the block must +// stay reserved without the lock held, and other nodes must keep allocating. +func TestTrimDiscardHoldsNoBitmapLock(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + lockA := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{1}, 0) + lockB := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{2}, 1) + + started := make(chan struct{}) + finish := make(chan struct{}) + var once sync.Once + tm := trim.NewManager(dev, trim.Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + once.Do(func() { close(started) }) + <-finish + return nil + }}) + a := NewRegion(dev, 0, 1000, lockA, [16]byte{1}) + a.SetTrimManager(tm) + tm.Start() + b := NewRegion(dev, 0, 1000, lockB, [16]byte{2}) + ctx := context.Background() + + blocks, err := a.Alloc(ctx, 1) + if err != nil { + t.Fatal(err) + } + if err := a.Free(ctx, blocks[0]); err != nil { + t.Fatal(err) + } + + stopped := make(chan error, 1) + go func() { stopped <- tm.Stop() }() + <-started + + if free, err := b.IsFree(blocks[0]); err != nil || free { + t.Errorf("block %d being discarded: IsFree = %v, %v; want reserved", blocks[0], free, err) + } + allocCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + theirs, err := b.Alloc(allocCtx, 1) + cancel() + if err != nil { + t.Fatalf("node B could not allocate during node A's discard: %v", err) + } + if theirs[0] == blocks[0] { + t.Errorf("node B allocated block %d while node A was discarding it", blocks[0]) + } + + close(finish) + if err := <-stopped; err != nil { + t.Fatal(err) + } + if free, err := a.IsFree(blocks[0]); err != nil || !free { + t.Errorf("block %d after discard: IsFree = %v, %v; want free again", blocks[0], free, err) + } +} + +// TestTrimReleasesReservationAfterSlowDiscard checks that the reserved blocks are +// freed even when the flush's context ran out during the discard and another +// node holds the bitmap lock at that moment. +func TestTrimReleasesReservationAfterSlowDiscard(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + lockA := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{1}, 0) + lockB := lock.NewManager(dev, 1<<20, 1<<20, [16]byte{2}, 1) + a := NewRegion(dev, 0, 1000, lockA, [16]byte{1}) + bg := context.Background() + + blocks, err := a.Alloc(bg, 1) + if err != nil { + t.Fatal(err) + } + if err := a.Free(bg, blocks[0]); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(bg, 100*time.Millisecond) + defer cancel() + err = a.withFreeBlocks(ctx, blocks, func([]uint64) { + if err := lockB.Acquire(bg, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + t.Errorf("node B lock: %v", err) + return + } + <-ctx.Done() + go func() { + time.Sleep(time.Second) + _ = lockB.Release(lock.LockTypeBlockBitmap, 0) + }() + }) + if err != nil { + t.Fatal(err) + } + + if free, err := a.IsFree(blocks[0]); err != nil || !free { + t.Errorf("block %d after a discard that outlasted the context: IsFree = %v, %v; want free", blocks[0], free, err) + } +} diff --git a/internal/cache/broadcast/broadcaster_test.go b/internal/cache/broadcast/broadcaster_test.go index 5413c748..94621ae5 100644 --- a/internal/cache/broadcast/broadcaster_test.go +++ b/internal/cache/broadcast/broadcaster_test.go @@ -98,6 +98,7 @@ func TestBroadcasterDisabled(t *testing.T) { } func TestBroadcasterDuplication(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -113,27 +114,28 @@ func TestBroadcasterDuplication(t *testing.T) { seq2 := uint64(2000) // First time should not be duplicate - if b.isDuplicate(seq1) { + if b.isDuplicate(peer, seq1) { t.Error("First check of seq1 should not be duplicate") } // Second time should be duplicate - if !b.isDuplicate(seq1) { + if !b.isDuplicate(peer, seq1) { t.Error("Second check of seq1 should be duplicate") } // Different sequence should not be duplicate - if b.isDuplicate(seq2) { + if b.isDuplicate(peer, seq2) { t.Error("First check of seq2 should not be duplicate") } // Original sequence still duplicate - if !b.isDuplicate(seq1) { + if !b.isDuplicate(peer, seq1) { t.Error("Third check of seq1 should still be duplicate") } } func TestBroadcasterCleanupSeen(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -146,7 +148,7 @@ func TestBroadcasterCleanupSeen(t *testing.T) { // Add some entries for i := uint64(1); i <= 10; i++ { - b.isDuplicate(i) + b.isDuplicate(peer, i) } b.seenMu.Lock() @@ -424,6 +426,7 @@ func TestConfigDisabled(t *testing.T) { // TestConcurrentDeduplication tests thread-safety of deduplication. func TestConcurrentDeduplication(t *testing.T) { + peer := [16]byte{9} handler := newMockHandler() uuid := [16]byte{1, 2, 3, 4} cfg := DefaultConfig() @@ -443,7 +446,7 @@ func TestConcurrentDeduplication(t *testing.T) { go func() { defer wg.Done() for j := 0; j < 100; j++ { - if b.isDuplicate(12345) { + if b.isDuplicate(peer, 12345) { atomic.AddInt64(&duplicates, 1) } } @@ -458,3 +461,28 @@ func TestConcurrentDeduplication(t *testing.T) { t.Errorf("Expected at least 990 duplicates, got %d", duplicates) } } + +// TestSameSequenceFromTwoSenders checks that dedup is per sender. Each node's +// sequence starts at its boot time in nanoseconds, so ranges can overlap. +func TestSameSequenceFromTwoSenders(t *testing.T) { + handler := newMockHandler() + cfg := DefaultConfig() + cfg.Enabled = false + + b, err := NewBroadcaster(cfg, [16]byte{1}, 0, handler) + if err != nil { + t.Fatalf("NewBroadcaster failed: %v", err) + } + + for i, sender := range [][16]byte{{2}, {3}} { + msg := NewMessageBuilder(sender).InvalidateInode(uint32(10+i), 1, 1) + msg.Sequence = 777 + b.handleMessage(msg, nil) + } + + handler.mu.Lock() + defer handler.mu.Unlock() + if len(handler.inodeInvalidated) != 2 { + t.Errorf("invalidated inodes = %v, want both senders' inodes [10 11]", handler.inodeInvalidated) + } +} diff --git a/internal/cache/broadcast/integrity_test.go b/internal/cache/broadcast/integrity_test.go new file mode 100644 index 00000000..2b8a55c7 --- /dev/null +++ b/internal/cache/broadcast/integrity_test.go @@ -0,0 +1,111 @@ +package broadcast + +import ( + "bytes" + "testing" +) + +func signedInvalidation(t *testing.T, secret []byte) []byte { + t.Helper() + msg := NewMessageBuilder([16]byte{0xAB}).InvalidateInode(42, 7, 9) + data, err := msg.Marshal(secret) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestHMACCoversEveryByte(t *testing.T) { + secret := []byte("cluster-secret") + data := signedInvalidation(t, secret) + + for i := range data { + for _, bit := range []byte{0x01, 0x80} { + tampered := bytes.Clone(data) + tampered[i] ^= bit + if _, err := Unmarshal(tampered, secret); err == nil { + t.Errorf("flipping bit %#x of byte %d was accepted", bit, i) + } + } + } + + if _, err := Unmarshal(append(bytes.Clone(data), 0), secret); err == nil { + t.Error("a trailing byte outside the HMAC was accepted") + } + + stripped := bytes.Clone(data[:len(data)-HMACSize]) + stripped[6] &^= FlagHMAC + if _, err := Unmarshal(stripped, secret); err == nil { + t.Error("a message with its HMAC stripped was accepted") + } +} + +func TestTruncatedMessageRejected(t *testing.T) { + for _, secret := range [][]byte{nil, []byte("cluster-secret")} { + data := signedInvalidation(t, secret) + for n := 0; n < len(data); n++ { + if msg, err := Unmarshal(data[:n], secret); err == nil { + t.Errorf("secret=%t: %d of %d bytes parsed as %+v", secret != nil, n, len(data), msg) + } + } + } +} + +// TestLargestMessageFitsOneDatagram checks that no message needs IP +// fragmentation on a 1500-byte MTU, where one lost fragment drops the message. +func TestLargestMessageFitsOneDatagram(t *testing.T) { + const ipv6UDPPayload = 1500 - 40 - 8 + if MaxMessageSize > ipv6UDPPayload { + t.Fatalf("MaxMessageSize %d exceeds the %d-byte UDP payload of a 1500-byte IPv6 MTU", MaxMessageSize, ipv6UDPPayload) + } + + largest := &Message{Version: ProtocolVersion, Type: TypeInvalidateInode, Payload: make([]byte, MaxPayloadSize)} + data, err := largest.Marshal([]byte("cluster-secret")) + if err != nil { + t.Fatalf("largest allowed message failed to marshal: %v", err) + } + if len(data) > MaxMessageSize { + t.Errorf("largest message is %d bytes, over MaxMessageSize %d", len(data), MaxMessageSize) + } + + tooBig := &Message{Version: ProtocolVersion, Type: TypeInvalidateInode, Payload: make([]byte, MaxPayloadSize+1)} + if _, err := tooBig.Marshal(nil); err == nil { + t.Error("payload over MaxPayloadSize was accepted") + } +} + +func FuzzUnmarshal(f *testing.F) { + secret := []byte("cluster-secret") + builder := NewMessageBuilder([16]byte{1}) + for _, msg := range []*Message{ + builder.InvalidateInode(1, 2, 3), + builder.InvalidateExtent(4, 5), + builder.InvalidateAll(), + builder.Heartbeat(), + builder.PeerAnnounce(1, 31337, [16]byte{}), + } { + for _, s := range [][]byte{nil, secret} { + data, err := msg.Marshal(s) + if err != nil { + f.Fatal(err) + } + f.Add(data) + } + } + + f.Fuzz(func(t *testing.T, data []byte) { + for _, s := range [][]byte{nil, secret} { + msg, err := Unmarshal(data, s) + if err != nil { + continue + } + again, err := msg.Marshal(s) + if err != nil { + t.Fatalf("accepted message does not marshal: %v", err) + } + if s != nil && !bytes.Equal(again, data) { + t.Fatalf("accepted bytes differ from the authenticated message:\n got %x\nwant %x", data, again) + } + } + }) +} diff --git a/internal/cache/broadcast/protocol.go b/internal/cache/broadcast/protocol.go index 377c5075..f967330f 100644 --- a/internal/cache/broadcast/protocol.go +++ b/internal/cache/broadcast/protocol.go @@ -209,6 +209,10 @@ func Unmarshal(data []byte, secret []byte) (*Message, error) { if len(data) < expectedSize { return nil, ErrMessageTooShort } + if len(data) > expectedSize { + // Trailing bytes would not be covered by the HMAC + return nil, ErrMessageTooLong + } // Verify HMAC if present and secret provided if hasHMAC { diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index e3323ee1..71d43f91 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -33,7 +33,7 @@ type Broadcaster struct { // Deduplication seenMu sync.RWMutex - seen map[uint64]time.Time // sequence -> first seen time + seen map[seenKey]time.Time // first seen time // Stats invalidationsReceived uint64 @@ -48,6 +48,13 @@ type Broadcaster struct { running atomic.Bool } +// seenKey identifies a message. Each node's sequence starts at its boot time in +// nanoseconds, so two nodes' sequence ranges can overlap. +type seenKey struct { + sender [16]byte + seq uint64 +} + // NewBroadcaster creates a new cache invalidation broadcaster. func NewBroadcaster(cfg Config, localUUID [16]byte, nodeSlot uint32, handler InvalidationHandler) (*Broadcaster, error) { transport, err := NewTransport(cfg) @@ -62,7 +69,7 @@ func NewBroadcaster(cfg Config, localUUID [16]byte, nodeSlot uint32, handler Inv localUUID: localUUID, nodeSlot: nodeSlot, localPort: uint16(cfg.UnicastPort), - seen: make(map[uint64]time.Time), + seen: make(map[seenKey]time.Time), } return b, nil @@ -132,7 +139,7 @@ func (b *Broadcaster) handleMessage(msg *Message, addr *net.UDPAddr) { } // Deduplicate - if b.isDuplicate(msg.Sequence) { + if b.isDuplicate(msg.SenderUUID, msg.Sequence) { atomic.AddUint64(&b.duplicatesDropped, 1) return } @@ -226,16 +233,17 @@ func (b *Broadcaster) handlePeerAnnounce(msg *Message, addr *net.UDPAddr) { payload.NodeSlot, uuidStr[:16], peerAddr) } -// isDuplicate checks if we've seen this sequence number recently. -func (b *Broadcaster) isDuplicate(seq uint64) bool { +// isDuplicate checks if we've seen this message from this sender recently. +func (b *Broadcaster) isDuplicate(sender [16]byte, seq uint64) bool { b.seenMu.Lock() defer b.seenMu.Unlock() - if _, exists := b.seen[seq]; exists { + key := seenKey{sender: sender, seq: seq} + if _, exists := b.seen[key]; exists { return true } - b.seen[seq] = time.Now() + b.seen[key] = time.Now() return false } @@ -281,9 +289,9 @@ func (b *Broadcaster) cleanupSeen() { defer b.seenMu.Unlock() cutoff := time.Now().Add(-5 * time.Minute) - for seq, seen := range b.seen { + for key, seen := range b.seen { if seen.Before(cutoff) { - delete(b.seen, seq) + delete(b.seen, key) } } } diff --git a/internal/trim/trim.go b/internal/trim/trim.go index d7c13eb0..75e1c43f 100644 --- a/internal/trim/trim.go +++ b/internal/trim/trim.go @@ -11,6 +11,7 @@ package trim import ( + "context" "log" "sort" "sync" @@ -50,8 +51,18 @@ type Config struct { // Enabled controls whether TRIM operations are actually issued. // When false, blocks are still tracked but no ioctl is issued. Enabled bool + + // Discard replaces the BLKDISCARD ioctl when set. Offsets and lengths are in bytes. + Discard func(offset, length uint64) error } +// FreeGuard calls discard with the queued blocks that are still free and keeps +// them from being allocated until discard returns. +type FreeGuard = func(ctx context.Context, blocks []uint64, discard func(free []uint64)) error + +// guardTimeout bounds how long a flush waits for the guard's allocation lock. +const guardTimeout = 30 * time.Second + // DefaultConfig returns the default TRIM configuration. func DefaultConfig() Config { return Config{ @@ -82,9 +93,11 @@ type Manager struct { // Pending blocks for TRIM pending []pendingBlock + guard FreeGuard // Background loop control stopCh chan struct{} + flushCh chan struct{} flushWg sync.WaitGroup stopOnce sync.Once // Ensures Stop() is idempotent @@ -102,9 +115,17 @@ func NewManager(dev *storage.Device, cfg Config) *Manager { config: cfg, pending: make([]pendingBlock, 0, cfg.BatchSize), stopCh: make(chan struct{}), + flushCh: make(chan struct{}, 1), } } +// SetFreeGuard makes every flush discard only the blocks the guard reports as free. +func (m *Manager) SetFreeGuard(guard FreeGuard) { + m.mu.Lock() + defer m.mu.Unlock() + m.guard = guard +} + // Start begins the background flush loop. func (m *Manager) Start() { m.flushWg.Add(1) @@ -120,17 +141,11 @@ func (m *Manager) Stop() error { close(m.stopCh) m.flushWg.Wait() - // Final flush of any remaining blocks - m.mu.Lock() - defer m.mu.Unlock() - - if len(m.pending) > 0 { - log.Printf("[trim] final flush of %d pending blocks", len(m.pending)) - m.flushLocked() - } + m.flush("final") + stats := m.Stats() log.Printf("[trim] stopped (queued=%d, discarded=%d, calls=%d)", - m.blocksQueued, m.blocksDiscarded, m.discardCalls) + stats.BlocksQueued, stats.BlocksDiscarded, stats.DiscardCalls) }) return nil } @@ -146,11 +161,18 @@ func (m *Manager) Queue(blockNum uint64) { queuedAt: time.Now(), }) m.blocksQueued++ + m.wakeIfFullLocked() +} - // Check if batch size threshold reached - if len(m.pending) >= m.config.BatchSize { - log.Printf("[trim] batch threshold reached (%d blocks)", len(m.pending)) - m.flushLocked() +// wakeIfFullLocked wakes the flush loop. Queue runs inside bitmap.Free, which +// holds the bitmap lock the guard needs, so it must not flush itself. +func (m *Manager) wakeIfFullLocked() { + if len(m.pending) < m.config.BatchSize { + return + } + select { + case m.flushCh <- struct{}{}: + default: } } @@ -168,11 +190,7 @@ func (m *Manager) QueueRange(startBlock, count uint64) { }) m.blocksQueued++ } - - if len(m.pending) >= m.config.BatchSize { - log.Printf("[trim] batch threshold reached (%d blocks)", len(m.pending)) - m.flushLocked() - } + m.wakeIfFullLocked() } // flushLoop runs periodically to check age-based flush conditions. @@ -187,6 +205,8 @@ func (m *Manager) flushLoop() { select { case <-m.stopCh: return + case <-m.flushCh: + m.flush("batch threshold") case <-ticker.C: m.checkAndFlush() } @@ -196,41 +216,56 @@ func (m *Manager) flushLoop() { // checkAndFlush flushes if the oldest pending block exceeds MaxAge. func (m *Manager) checkAndFlush() { m.mu.Lock() - defer m.mu.Unlock() - - if len(m.pending) == 0 { - return - } - - // Find oldest entry - oldest := m.pending[0].queuedAt - for _, p := range m.pending { - if p.queuedAt.Before(oldest) { - oldest = p.queuedAt + due := false + if len(m.pending) > 0 { + oldest := m.pending[0].queuedAt + for _, p := range m.pending { + if p.queuedAt.Before(oldest) { + oldest = p.queuedAt + } } + due = time.Since(oldest) >= m.config.MaxAge } + m.mu.Unlock() - if time.Since(oldest) >= m.config.MaxAge { - log.Printf("[trim] age threshold reached (%d blocks, oldest=%v)", - len(m.pending), time.Since(oldest)) - m.flushLocked() + if due { + m.flush("age threshold") } } -// flushLocked issues BLKDISCARD for all pending blocks. -// Must be called with m.mu held. -func (m *Manager) flushLocked() { - if len(m.pending) == 0 { +// flush discards the pending blocks. The guard runs without m.mu held, because +// Free takes the bitmap lock before m.mu. +func (m *Manager) flush(reason string) { + m.mu.Lock() + blocks := make([]uint64, len(m.pending)) + for i, p := range m.pending { + blocks[i] = p.blockNum + } + m.pending = m.pending[:0] + guard := m.guard + m.mu.Unlock() + + if len(blocks) == 0 { return } + log.Printf("[trim] %s: flushing %d blocks", reason, len(blocks)) - // Coalesce adjacent blocks into ranges - ranges := m.coalesceBlocks() + if guard == nil { + m.discardBlocks(blocks) + return + } + ctx, cancel := context.WithTimeout(context.Background(), guardTimeout) + defer cancel() + if err := guard(ctx, blocks, m.discardBlocks); err != nil { + log.Printf("[trim] skipping discard of %d blocks: %v", len(blocks), err) + } +} - log.Printf("[trim] flushing %d blocks in %d ranges", len(m.pending), len(ranges)) +// discardBlocks issues one discard per contiguous range of blocks. +func (m *Manager) discardBlocks(blocks []uint64) { + ranges := coalesceBlocks(blocks) - // Issue BLKDISCARD for each range - discarded := uint64(0) + var discarded, calls uint64 for _, r := range ranges { if err := m.issueDiscard(r); err != nil { log.Printf("[trim] discard failed for range [%d, %d): %v", @@ -238,40 +273,39 @@ func (m *Manager) flushLocked() { continue } discarded += r.count - m.discardCalls++ + calls++ } + m.mu.Lock() + defer m.mu.Unlock() m.blocksDiscarded += discarded + m.discardCalls += calls if len(ranges) > 0 { m.coalesceRatio = float64(discarded) / float64(len(ranges)) } - - // Clear pending list - m.pending = m.pending[:0] } -// coalesceBlocks groups adjacent blocks into contiguous ranges. -// Returns sorted ranges for efficient BLKDISCARD. -func (m *Manager) coalesceBlocks() []blockRange { - if len(m.pending) == 0 { +// coalesceBlocks sorts blocks, drops duplicates and groups adjacent blocks into +// contiguous ranges. +func coalesceBlocks(blocks []uint64) []blockRange { + if len(blocks) == 0 { return nil } - // Sort blocks by block number - sort.Slice(m.pending, func(i, j int) bool { - return m.pending[i].blockNum < m.pending[j].blockNum - }) + sort.Slice(blocks, func(i, j int) bool { return blocks[i] < blocks[j] }) var ranges []blockRange current := blockRange{ - startBlock: m.pending[0].blockNum, + startBlock: blocks[0], count: 1, } - for i := 1; i < len(m.pending); i++ { - blockNum := m.pending[i].blockNum + for i := 1; i < len(blocks); i++ { + blockNum := blocks[i] - // Check if adjacent to current range + if blockNum < current.startBlock+current.count { + continue + } if blockNum == current.startBlock+current.count { current.count++ } else { @@ -303,6 +337,9 @@ func (m *Manager) issueDiscard(r blockRange) error { // startBlock is relative to data region, add DataOffset byteOffset := m.config.DataOffset + r.startBlock*BlockSize byteLength := r.count * BlockSize + if m.config.Discard != nil { + return m.config.Discard(byteOffset, byteLength) + } // BLKDISCARD takes [2]uint64{offset, length} args := [2]uint64{byteOffset, byteLength} diff --git a/internal/trim/trim_test.go b/internal/trim/trim_test.go index 16d1dce7..fffdd481 100644 --- a/internal/trim/trim_test.go +++ b/internal/trim/trim_test.go @@ -9,18 +9,7 @@ import ( ) func TestCoalesceBlocks(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 10, queuedAt: time.Now()}, - {blockNum: 11, queuedAt: time.Now()}, - {blockNum: 12, queuedAt: time.Now()}, - {blockNum: 20, queuedAt: time.Now()}, - {blockNum: 21, queuedAt: time.Now()}, - {blockNum: 50, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{10, 11, 12, 20, 21, 50}) if len(ranges) != 3 { t.Fatalf("expected 3 ranges, got %d", len(ranges)) @@ -46,24 +35,14 @@ func TestCoalesceBlocks(t *testing.T) { } func TestCoalesceBlocksEmpty(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{}, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks(nil) if ranges != nil { t.Errorf("expected nil for empty pending, got %v", ranges) } } func TestCoalesceBlocksSingleBlock(t *testing.T) { - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 42, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{42}) if len(ranges) != 1 { t.Fatalf("expected 1 range, got %d", len(ranges)) } @@ -75,15 +54,7 @@ func TestCoalesceBlocksSingleBlock(t *testing.T) { func TestCoalesceBlocksUnsorted(t *testing.T) { // Blocks out of order - should still coalesce correctly - m := &Manager{ - pending: []pendingBlock{ - {blockNum: 12, queuedAt: time.Now()}, - {blockNum: 10, queuedAt: time.Now()}, - {blockNum: 11, queuedAt: time.Now()}, - }, - } - - ranges := m.coalesceBlocks() + ranges := coalesceBlocks([]uint64{12, 10, 11}) if len(ranges) != 1 { t.Fatalf("expected 1 range (after sorting), got %d", len(ranges)) } @@ -352,3 +323,27 @@ func TestStatsAreThreadSafe(t *testing.T) { t.Errorf("expected 100 queued, got %d", stats.BlocksQueued) } } + +func TestDuplicateBlockDiscardedOnce(t *testing.T) { + dev, cleanup := createTestDevice(t) + defer cleanup() + + var calls [][2]uint64 + m := NewManager(dev, Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + calls = append(calls, [2]uint64{offset, length}) + return nil + }}) + m.Start() + + m.Queue(5) + m.Queue(5) + m.Queue(6) + if err := m.Stop(); err != nil { + t.Fatal(err) + } + + want := [][2]uint64{{5 * BlockSize, 2 * BlockSize}} + if len(calls) != 1 || calls[0] != want[0] { + t.Errorf("discards = %v, want %v", calls, want) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index df005977..28803bfd 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1091,6 +1091,55 @@ scenarios: environments: go: internal/trim/trim_test.go::TestStatsAreThreadSafe + - id: trim-skip-reallocated + name: TRIM skips a reallocated block + description: A block freed and allocated again before the flush is not discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimSkipsBlockReallocatedBeforeFlush + + - id: trim-skip-remote-allocated + name: TRIM skips a block another node allocated + description: A freed block another node allocated before the flush is not discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimSkipsBlockAnotherNodeAllocated + + - id: trim-discard-free + name: TRIM discards free blocks + description: Blocks still free at flush time are discarded + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimDiscardsFreedBlocks + + - id: trim-batch-flush-from-free + name: TRIM batch flush from Free + description: A full batch flushes in the background without deadlocking Free on the bitmap lock + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimBatchFlushFromFree + + - id: trim-discard-without-lock + name: TRIM discard without the bitmap lock + description: With real locks on two nodes, a block being discarded stays reserved and the other node keeps allocating + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimDiscardHoldsNoBitmapLock + + - id: trim-release-after-slow-discard + name: TRIM frees reserved blocks after a slow discard + description: Reserved blocks are freed even when the flush context ran out during the discard and another node holds the bitmap lock + priority: go_only + environments: + go: internal/bitmap/trim_test.go::TestTrimReleasesReservationAfterSlowDiscard + + - id: trim-duplicate-block + name: TRIM duplicate block + description: A block queued twice is discarded once + priority: go_only + environments: + go: internal/trim/trim_test.go::TestDuplicateBlockDiscardedOnce + # =========================================================================== # SPACE REPORTING TESTS (Phase 5: Thin Provisioning) # =========================================================================== @@ -1458,6 +1507,41 @@ scenarios: environments: go: internal/cache/broadcast/broadcaster_test.go::TestConfigDefaults + - id: broadcast-dedup-per-sender + name: Broadcast dedup per sender + description: The same sequence number from two senders is processed twice + priority: go_only + environments: + go: internal/cache/broadcast/broadcaster_test.go::TestSameSequenceFromTwoSenders + + - id: broadcast-hmac-every-byte + name: Broadcast HMAC covers every byte + description: Flipping any bit, appending a byte or stripping the HMAC is rejected + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestHMACCoversEveryByte + + - id: broadcast-truncation + name: Broadcast truncated messages + description: Every truncation of a message is rejected + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestTruncatedMessageRejected + + - id: broadcast-single-datagram + name: Broadcast message fits one datagram + description: The largest message fits a 1500-byte MTU without fragmentation + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::TestLargestMessageFitsOneDatagram + + - id: broadcast-fuzz-unmarshal + name: Broadcast Unmarshal fuzz + description: Accepted signed datagrams are exactly what Marshal produces + priority: go_only + environments: + go: internal/cache/broadcast/integrity_test.go::FuzzUnmarshal + # --------------------------------------------------------------------------- # Broadcast Integration Tests (end-to-end cross-node invalidation) # ---------------------------------------------------------------------------