From 237dc8da7d6c3c2558cbbb0f28c3d8fe921668e7 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 00:26:43 -0600 Subject: [PATCH 1/7] feat(alloc): hand writes data blocks from a per-node reserve Each node takes up to 64 blocks from the bitmap under one lock hold and hands them to writes without the global bitmap lock. The reserve is a journal record, so replay and fsck free a crashed node's unused blocks. An idle reserve goes back after 10s, and a node that finds the bitmap full asks peers for their reserves and waits up to 10s before ENOSPC. No reserves below max(1%, 2 x size x live nodes) free blocks. New -block-reserve flag (0 turns it off) and reserve metrics. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + cmd/goclusterfs-mount/main.go | 23 +- docs/API-CONFIG.md | 3 +- docs/OPERATIONS.md | 10 + internal/api/prometheus.go | 44 ++ internal/api/prometheus_test.go | 6 + internal/api/types.go | 10 + internal/bitmap/bitmap.go | 83 ++++ internal/cache/broadcast/protocol.go | 17 + internal/cache/broadcast/receiver.go | 22 + internal/cache/cache.go | 14 + internal/extent/tree_edit.go | 4 + internal/goclusterfs/fs.go | 152 ++++++- .../goclusterfs/reserve_power_cut_test.go | 144 +++++++ internal/goclusterfs/reserve_test.go | 264 ++++++++++++ internal/journal/journal.go | 46 +++ internal/reserve/reserve.go | 377 ++++++++++++++++++ internal/reserve/reserve_test.go | 178 +++++++++ internal/stats/sampler.go | 10 + tests/test-manifest.yaml | 121 ++++++ tests/vm/scripts/run-vm-tests.sh | 56 +++ 21 files changed, 1560 insertions(+), 25 deletions(-) create mode 100644 internal/goclusterfs/reserve_power_cut_test.go create mode 100644 internal/goclusterfs/reserve_test.go create mode 100644 internal/reserve/reserve.go create mode 100644 internal/reserve/reserve_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1136020b..00a875cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Read a snapshot by cloning it with `user.snapshot.clone`. - `goclusterfs-fsck` consistency checker. `-repair` rebuilds block refcounts from every file, snapshot and clone. It refuses to run while the device is in use, and `-repair -dry-run` shows what it would change. Exit codes follow fsck(8): `0` no errors, `1` repaired, `4` errors remain, `8` failed or refused, `16` usage error. - Benchmark suite (`tests/benchmark`) and an `internal/metrics` package. +- Per-node block reserves (`-block-reserve`, default 64): each node takes a batch of free blocks under one hold of the bitmap lock and hands them to its writes without it, gives unused ones back when idle, at unmount and when space runs low, and waits up to 10 seconds for other nodes' reserves before reporting `ENOSPC`. Journal replay and fsck return the unused reserve of a crashed node. See OPERATIONS.md. - `/metrics` and `/api/v1/stats/lock` report, per lock type, the acquires served, the acquires that found the lock taken, and how long those waited (`goclusterfs_lock_acquires_total`, `goclusterfs_lock_blocked_total`, `goclusterfs_lock_wait_seconds_total`). - Go, Docker, and QEMU VM test suites, with `tests/test-manifest.yaml` and `make test-parity` to track coverage across them. - Fuzz tests for the on-disk decoders (superblock, inode, snapshot descriptor, journal superblock and records) and the journal recovery scan. diff --git a/cmd/goclusterfs-mount/main.go b/cmd/goclusterfs-mount/main.go index c634c815..10f0ec37 100644 --- a/cmd/goclusterfs-mount/main.go +++ b/cmd/goclusterfs-mount/main.go @@ -33,6 +33,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/logging" "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/refcount" + "github.com/supporttools/GoClusterFS/internal/reserve" "github.com/supporttools/GoClusterFS/internal/stats" "github.com/supporttools/GoClusterFS/internal/storage" "github.com/supporttools/GoClusterFS/internal/superblock" @@ -111,6 +112,7 @@ func main() { // TRIM configuration trimEnabled = flag.Bool("trim", true, "Discard freed blocks on the device (BLKDISCARD) in batches; does nothing on image files") trimBatchSize = flag.Int("trim-batch", 256, "Freed blocks to collect before a discard") + blockReserve = flag.Int("block-reserve", reserve.DefaultSize, "Data blocks each node takes from the bitmap at once and hands out without the cluster-wide bitmap lock; 0 turns reserves off. Needs the journal") trimMaxAge = flag.Duration("trim-age", 5*time.Second, "Longest a freed block waits for its discard") // SFB configuration (Sub-File Block containers for small files) @@ -434,6 +436,20 @@ func main() { // Create filesystem with journal, TRIM, and refcount for crash consistency gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, nil, refcountReg, lockMgr, nodeUUID, dataOffset) + var res *reserve.Manager + if j != nil && *blockReserve > 0 { + res = reserve.New(reserve.Config{Size: *blockReserve}, bm, j, extentMgr.ReleaseBlocks, func() int { + active, _ := nodeReg.GetActiveNodes() + return len(active) + }) + res.SetBroadcast(cacheMgr.BroadcastReturnReserves) + cacheMgr.OnReturnReserves(func() { res.ReturnAll(context.Background()) }) + bm.SetNoSpaceWait(res) + j.SetCheckpointHook(func() { res.Refresh(context.Background()) }) + gfs.SetReserve(res) + res.Start() + log.Printf("Block reserve: up to %d blocks per node", *blockReserve) + } // Set up cross-node recovery handler for failed node journal replay recoveryCtx := &journal.RecoveryContext{ @@ -506,7 +522,7 @@ func main() { if err != nil { log.Printf("Warning: Failed to create the API server: %v; the mount continues without it", err) } else { - sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, dirMgr, j, trimMgr, bm, nodeReg)) + sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, dirMgr, j, trimMgr, bm, nodeReg, res)) apiServer.SetStatusProvider(&fsStatusProvider{ mountPath: *mountPoint, devicePath: *devicePath, @@ -653,8 +669,11 @@ func usageFailf(format string, args ...any) { os.Exit(cli.ExitUsage) } -func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, dirMgr *dir.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry) stats.Sources { +func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, dirMgr *dir.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry, res *reserve.Manager) stats.Sources { src := stats.Sources{Cache: cacheMgr, Locks: lockMgr, Dirs: dirMgr, Blocks: bm, Nodes: nodeReg, BlockSize: goclusterfs.BlockSize} + if res != nil { + src.Reserve = res + } if j != nil { src.Journal = j } diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 69be214a..77294d6e 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -64,6 +64,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For |------|------|---------|-------------| | `-journal` | bool | `true` | Write-ahead journal for crash consistency. Off, a crash of this node can leave the filesystem inconsistent: testing only | | `-skip-recovery` | int | `-1` (off) | Discard the journal of registry slot N instead of replaying it, if the mount takes slot N (dangerous, may lose data) | +| `-block-reserve` | int | `64` | Data blocks each node takes from the bitmap at once and hands out to writes without the cluster-wide bitmap lock; `0` turns reserves off. Needs `-journal`. See [Block Reserves](OPERATIONS.md#block-reserves) | ### Network Broadcast Flags @@ -166,7 +167,7 @@ Mount copies its managers' counters every second. While requests read capacity o | `/api/v1/stats/sfb` | 404, because SFB containers are disabled | | `/api/v1/stats/all` | `collected_at` and every section above except `sfb` | -`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics. +`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics. --- diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 6796ca6a..c7393759 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -306,6 +306,16 @@ If a node did stall for 15 seconds or more while mounted (hung-task warnings in What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks as described under [Node Requirements](#node-requirements). +### Block Reserves + +Each node takes up to 64 free blocks from the block bitmap at once (`-block-reserve`), under one hold of the cluster-wide bitmap lock, and hands them to its writes without that lock. Without reserves, every new data block takes the bitmap lock, and with four nodes writing at once about 60% of their time went to waiting for it. Only new data blocks of writes come from the reserve. Extent-tree blocks, directory blocks, copy-on-write copies and `fallocate` still take the bitmap lock. + +- **Size.** A reserve holds at most `-block-reserve` blocks, and at most 1/16 of the free blocks per mounted node. There are no reserves while fewer blocks are free than 1% of the device or two reserves per mounted node. +- **Going back.** A node gives its unused reserve back after 10 seconds without a write, at a clean unmount, and when a write finds free space below that line. +- **Full device.** A node that finds no free block gives back its own reserve, asks the other nodes over broadcast to give back theirs, and retries for up to 10 seconds before it fails with `ENOSPC`. It logs the wait at INFO, and `goclusterfs_reserve_peer_waits_total` and `goclusterfs_reserve_peer_wait_seconds_total` count it. Without broadcast, other nodes still give back idle reserves within 10 seconds. +- **`df`.** Reserved blocks count as used until a write uses them or the node gives them back: at most 64 MB per node at the default. +- **Crash.** Each reserve is a journal record. When the node's journal is replayed, by the node at its next mount or by another node, the blocks it reserved and never used go back to the bitmap. fsck reports them as "reserved by node N and never used", which is not an error, and `fsck -repair -force` frees them after it marks the reserve done in that node's journal, so a later replay does not free them again. A crash in a few narrow windows, such as between ending a reserve and freeing it, leaks a few blocks instead; `fsck -repair` reclaims them. + ### Handling Node Failure When a node fails unexpectedly: diff --git a/internal/api/prometheus.go b/internal/api/prometheus.go index d51d0b30..0750034f 100644 --- a/internal/api/prometheus.go +++ b/internal/api/prometheus.go @@ -97,6 +97,12 @@ type fsCollector struct { lockBlockedDesc *prometheus.Desc lockWaitSecondsDesc *prometheus.Desc + reserveBlocksDesc *prometheus.Desc + reserveRefillsDesc *prometheus.Desc + reserveReturnsDesc *prometheus.Desc + reservePeerWaitsDesc *prometheus.Desc + reservePeerWaitSecondsDesc *prometheus.Desc + // SFB (Sparse File Bitmap) metrics sfbTotalBlocksDesc *prometheus.Desc sfbAllocatedBlocksDesc *prometheus.Desc @@ -395,6 +401,31 @@ func newFSCollector(s *Server) *fsCollector { "Time blocked lock acquires spent waiting, by lock type", []string{"type"}, nil, ), + reserveBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "reserve", "blocks"), + "Data blocks this node holds in reserve and has not handed out", + nil, nil, + ), + reserveRefillsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "reserve", "refills_total"), + "Reserves this node took from the bitmap", + nil, nil, + ), + reserveReturnsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "reserve", "returns_total"), + "Reserves this node gave back to the bitmap", + nil, nil, + ), + reservePeerWaitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "reserve", "peer_waits_total"), + "Allocations that found the bitmap full and waited for other nodes to give back reserved blocks", + nil, nil, + ), + reservePeerWaitSecondsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "reserve", "peer_wait_seconds_total"), + "Time allocations waited for other nodes' reserved blocks", + nil, nil, + ), // SFB (Sparse File Bitmap) sfbTotalBlocksDesc: prometheus.NewDesc( @@ -535,6 +566,11 @@ func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.lockAcquiresDesc ch <- c.lockBlockedDesc ch <- c.lockWaitSecondsDesc + ch <- c.reserveBlocksDesc + ch <- c.reserveRefillsDesc + ch <- c.reserveReturnsDesc + ch <- c.reservePeerWaitsDesc + ch <- c.reservePeerWaitSecondsDesc // SFB ch <- c.sfbTotalBlocksDesc @@ -594,6 +630,14 @@ func (c *fsCollector) Collect(ch chan<- prometheus.Metric) { if lock := provider.LockStats(); lock != nil { c.collectLockStats(ch, lock) } + if all := provider.AllStats(); all != nil && all.Reserve != nil { + r := all.Reserve + ch <- prometheus.MustNewConstMetric(c.reserveBlocksDesc, prometheus.GaugeValue, float64(r.Blocks)) + ch <- prometheus.MustNewConstMetric(c.reserveRefillsDesc, prometheus.CounterValue, float64(r.Refills)) + ch <- prometheus.MustNewConstMetric(c.reserveReturnsDesc, prometheus.CounterValue, float64(r.Returns)) + ch <- prometheus.MustNewConstMetric(c.reservePeerWaitsDesc, prometheus.CounterValue, float64(r.PeerWaits)) + ch <- prometheus.MustNewConstMetric(c.reservePeerWaitSecondsDesc, prometheus.CounterValue, r.PeerWaitSeconds) + } // Collect SFB stats if sfb := provider.SFBStats(); sfb != nil { diff --git a/internal/api/prometheus_test.go b/internal/api/prometheus_test.go index a4d70ba5..9fb402a2 100644 --- a/internal/api/prometheus_test.go +++ b/internal/api/prometheus_test.go @@ -57,6 +57,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { // Set up mock provider with all stats server.SetStatsProvider(&mockStatsProvider{ + all: &AllStatsResponse{Reserve: &ReserveStatsResponse{Blocks: 7, Refills: 3, Returns: 2, PeerWaits: 1, PeerWaitSeconds: 0.5}}, cache: &CacheStatsResponse{ InodeCache: CacheHitRateInfo{ Size: 1000, @@ -203,6 +204,11 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { `goclusterfs_lock_wait_seconds_total{type="bitmap"} 16.5`, `goclusterfs_lock_acquires_total{type="refcount"} 40`, `goclusterfs_lock_blocked_total{type="refcount"} 0`, + "goclusterfs_reserve_blocks 7", + "goclusterfs_reserve_refills_total 3", + "goclusterfs_reserve_returns_total 2", + "goclusterfs_reserve_peer_waits_total 1", + "goclusterfs_reserve_peer_wait_seconds_total 0.5", // SFB metrics "goclusterfs_sfb_total_blocks 1000", diff --git a/internal/api/types.go b/internal/api/types.go index cfe2318e..929c84ed 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -230,6 +230,16 @@ type AllStatsResponse struct { Lock *LockStatsResponse `json:"lock,omitempty"` SFB *SFBStatsResponse `json:"sfb,omitempty"` Cluster *ClusterInfoResponse `json:"cluster,omitempty"` + Reserve *ReserveStatsResponse `json:"reserve,omitempty"` +} + +// ReserveStatsResponse counts this node's block reserve since it mounted. +type ReserveStatsResponse struct { + Blocks int `json:"blocks"` + Refills uint64 `json:"refills"` + Returns uint64 `json:"returns"` + PeerWaits uint64 `json:"peer_waits"` + PeerWaitSeconds float64 `json:"peer_wait_seconds"` } // StatsOverviewResponse is returned by GET /api/v1/stats. diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index ade90421..7a4841f6 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -66,6 +66,18 @@ type Region struct { lockMgr *lock.Manager // For distributed locking (optional) nodeUUID [16]byte // This node's UUID for locking trimMgr TrimQueuer // Optional TRIM manager for batched discards + noSpace NoSpaceWait +} + +// NoSpaceWait lets Alloc wait for blocks that other nodes hold in reserve. +type NoSpaceWait interface { + WaitForPeers(ctx context.Context, waited time.Duration) bool + Waited(d time.Duration) +} + +// SetNoSpaceWait makes Alloc wait on w instead of failing when no block is free. +func (r *Region) SetNoSpaceWait(w NoSpaceWait) { + r.noSpace = w } // NewRegion creates a new bitmap region manager. @@ -89,6 +101,28 @@ func NewRegion(dev *storage.Device, offset, totalBlocks uint64, lockMgr *lock.Ma // Alloc allocates the requested number of blocks. // Returns the allocated block numbers, or an error if not enough free blocks. func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) { + var start time.Time + for { + blocks, err := r.allocOnce(ctx, count) + if !errors.Is(err, ErrNoFreeBlocks) || r.noSpace == nil { + if err == nil && !start.IsZero() { + r.noSpace.Waited(time.Since(start)) + } + return blocks, err + } + waited := time.Duration(0) + if start.IsZero() { + start = time.Now() + } else { + waited = time.Since(start) + } + if !r.noSpace.WaitForPeers(ctx, waited) { + return nil, err + } + } +} + +func (r *Region) allocOnce(ctx context.Context, count int) ([]uint64, error) { if count <= 0 { return nil, nil } @@ -332,6 +366,55 @@ func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { } } +// Reserve calls record before marking the blocks allocated, and returns them even if marking fails. +func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) error) ([]uint64, error) { + if r.lockMgr != nil { + if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + return nil, fmt.Errorf("failed to acquire bitmap lock: %w", err) + } + defer r.unlock() + } + r.mu.Lock() + defer r.mu.Unlock() + + numSectors := (r.totalBlocks + BitsPerSector - 1) / BitsPerSector + var picked, changed []uint64 + sectors := make(map[uint64][]byte) + for sectorIdx := uint64(0); sectorIdx < numSectors && len(picked) < n; sectorIdx++ { + data, err := r.readSector(sectorIdx) + if err != nil { + return nil, fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) + } + before := len(picked) + for bit := uint64(0); bit < BitsPerSector && len(picked) < n; bit++ { + block := sectorIdx*BitsPerSector + bit + if block >= r.totalBlocks { + break + } + if data[bit/8]&(1<<(bit%8)) == 0 { + data[bit/8] |= 1 << (bit % 8) + picked = append(picked, block) + } + } + if len(picked) > before { + sectors[sectorIdx] = data + changed = append(changed, sectorIdx) + } + } + if len(picked) == 0 { + return nil, ErrNoFreeBlocks + } + if err := record(picked); err != nil { + return nil, err + } + for _, sectorIdx := range changed { + if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil { + return picked, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + } + return picked, nil +} + // HoldAllocated takes the bitmap lock and returns the blocks of blocks that are // allocated. The lock stays held, so none of them can be freed, until release. // A block past the end of the volume is an error. diff --git a/internal/cache/broadcast/protocol.go b/internal/cache/broadcast/protocol.go index 80014055..699a0a5b 100644 --- a/internal/cache/broadcast/protocol.go +++ b/internal/cache/broadcast/protocol.go @@ -58,6 +58,9 @@ const ( // TypePeerAnnounce announces a node's presence and IP address. TypePeerAnnounce MessageType = 5 + + // TypeReturnReserves asks peers to give their reserved blocks back to the bitmap. + TypeReturnReserves MessageType = 6 ) // String returns a human-readable message type name. @@ -73,6 +76,8 @@ func (t MessageType) String() string { return "Heartbeat" case TypePeerAnnounce: return "PeerAnnounce" + case TypeReturnReserves: + return "ReturnReserves" default: return fmt.Sprintf("Unknown(%d)", t) } @@ -412,6 +417,18 @@ func (b *MessageBuilder) InvalidateAll() *Message { } } +// ReturnReserves creates a request for peers to give back their reserved blocks. +func (b *MessageBuilder) ReturnReserves() *Message { + return &Message{ + Version: ProtocolVersion, + Type: TypeReturnReserves, + Flags: FlagUrgent, + SenderUUID: b.senderUUID, + Sequence: b.seqGen.Next(), + Timestamp: time.Now().UnixNano(), + } +} + // Heartbeat creates a heartbeat message. func (b *MessageBuilder) Heartbeat() *Message { return &Message{ diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index cf3d7139..0c332740 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -33,6 +33,8 @@ type Broadcaster struct { sendMu sync.Mutex + onReturnReserves atomic.Pointer[func()] + sendersMu sync.Mutex senders map[[16]byte]*senderSeq @@ -186,6 +188,26 @@ func (b *Broadcaster) handleMessage(msg *Message, addr *net.UDPAddr) { case TypePeerAnnounce: b.handlePeerAnnounce(msg, addr) + + case TypeReturnReserves: + if fn := b.onReturnReserves.Load(); fn != nil { + go (*fn)() + } + } +} + +// OnReturnReserves sets what runs when a peer asks for reserved blocks back. +func (b *Broadcaster) OnReturnReserves(fn func()) { + b.onReturnReserves.Store(&fn) +} + +// A lost message only makes the asker wait: peers also return idle reserves on their own. +func (b *Broadcaster) BroadcastReturnReserves() { + if !b.running.Load() { + return + } + if err := b.send(b.builder.ReturnReserves); err != nil { + log.Printf("[broadcast] failed to send ReturnReserves: %v", err) } } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 2cce3021..8e83bacf 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -327,6 +327,20 @@ func (m *Manager) BroadcastInvalidateExtent(inodeNum uint32, dataGen uint64) { } } +// BroadcastReturnReserves asks peer nodes to give back their reserved blocks. +func (m *Manager) BroadcastReturnReserves() { + if m.broadcaster != nil { + m.broadcaster.BroadcastReturnReserves() + } +} + +// OnReturnReserves must be called after StartBroadcast. +func (m *Manager) OnReturnReserves(fn func()) { + if m.broadcaster != nil { + m.broadcaster.OnReturnReserves(fn) + } +} + // BroadcastInvalidateAll clears all caches locally AND broadcasts // a full invalidation to peer nodes. Use this after events that // might invalidate many entries (fsck, network partition recovery, etc). diff --git a/internal/extent/tree_edit.go b/internal/extent/tree_edit.go index f2cea438..dd354023 100644 --- a/internal/extent/tree_edit.go +++ b/internal/extent/tree_edit.go @@ -662,6 +662,10 @@ func (m *Manager) ReserveTreeBlocks(ctx context.Context, ino *inode.Inode) (cont return context.WithValue(ctx, reservationKey{}, r), release, nil } +func (m *Manager) TreeBlocksForInsert(ino *inode.Inode) (int, error) { + return m.treeBlocksForInsert(ino) +} + func (m *Manager) treeBlocksForInsert(ino *inode.Inode) (int, error) { leaves, err := m.layout(ino) if err != nil { diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 3dc4b3b4..a535805e 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -33,6 +33,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/logging" "github.com/supporttools/GoClusterFS/internal/refcount" + "github.com/supporttools/GoClusterFS/internal/reserve" "github.com/supporttools/GoClusterFS/internal/sfb" "github.com/supporttools/GoClusterFS/internal/snapshot" "github.com/supporttools/GoClusterFS/internal/storage" @@ -128,6 +129,7 @@ type FS struct { dirMgr *dir.Manager cacheMgr *cache.Manager // Inode cache for read optimization journal *journal.Journal // Per-node journal for crash consistency (WAL) + reserve *reserve.Manager // this node's batch of data blocks; nil without a journal trimMgr *trim.Manager // TRIM manager for batched block discards sfbMgr *sfb.Manager // SFB manager for small file storage (optional) refcountReg *refcount.Region // Reference count table for CoW/snapshots (optional) @@ -298,6 +300,13 @@ func (f *FS) InitSnapshots(ctx context.Context, snapshotOffset, snapshotSize uin return nil } +// SetReserve makes writes take data blocks from r. It needs the journal. +func (f *FS) SetReserve(r *reserve.Manager) { + if f.journal != nil { + f.reserve = r + } +} + // Close closes the filesystem and releases resources. func (f *FS) Close() error { // Stop snapshot worker first to complete pending jobs @@ -305,6 +314,11 @@ func (f *FS) Close() error { ShutdownStep("snapshot worker stop", f.snapshotWkr.Stop) } + // Unused reserved blocks go back before TRIM and the journal stop. + if f.reserve != nil { + ShutdownStep("block reserve return", func() { f.reserve.Stop(context.Background()) }) + } + // Stop TRIM manager to flush pending discards if f.trimMgr != nil { ShutdownStep("TRIM stop", func() { @@ -1896,39 +1910,76 @@ var testHookBeforeInodeCommit func() error // testHookBeforeFreeingUnused, when set by a test, runs after a failed write closes its intent and before it frees blocks. var testHookBeforeFreeingUnused func() +// mapReadyBlockLocked lets a write keep the inode lock when the reserve has a block and the tree needs none. +func (f *File) mapReadyBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64, lockHeld bool) (*extent.Extent, bool) { + if f.fs.reserve == nil || !lockHeld { + return nil, false + } + if need, err := f.fs.extentMgr.TreeBlocksForInsert(ino); err != nil || need > 0 { + return nil, false + } + b, ok := f.fs.reserve.TakeReady() + if !ok { + return nil, false + } + ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: b, Length: 1, Flags: extent.FlagAllocated} + if err := f.fs.extentMgr.Insert(ctx, ino, ext); err != nil { + f.fs.reserve.Untake(b) + return nil, false + } + ino.AllocatedBlocks += ext.Length + ino.WrittenBlocks += ext.Length + return ext, true +} + // allocateBlockLocked maps a new data block at blockNum and returns the inode // re-read under the inode lock, which stays held on success. The bitmap comes // before the inode in lock order, so the data block and any extent-tree blocks // are allocated before the lock is taken. Call the returned release after // dropping the inode lock. -func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, func(), error) { - blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) - if err != nil { - logging.Debugf("Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, err) - return nil, nil, nil, allocErrno(err) +func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockNum uint64) (*inode.Inode, *extent.Extent, func(), bool, error) { + physBlock, reserved := uint64(0), false + if f.fs.reserve != nil { + physBlock, reserved = f.fs.reserve.Take(ctx) } - ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: blocks[0], Length: 1, Flags: extent.FlagAllocated} + if !reserved { + blocks, err := f.fs.bitmapMgr.Alloc(ctx, 1) + if err != nil { + logging.Debugf("Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, err) + return nil, nil, nil, false, allocErrno(err) + } + physBlock = blocks[0] + } + // A reserved block goes back to the reserve: its record still lists it. + giveBack := func() { + if reserved { + f.fs.reserve.Untake(physBlock) + } else { + f.fs.freeUnused(ctx, f.inodeNum, physBlock) + } + } + ext := &extent.Extent{LogicalBlock: blockNum, PhysicalBlock: physBlock, Length: 1, Flags: extent.FlagAllocated} treeCtx, releaseTree, err := f.fs.extentMgr.ReserveTreeBlocks(ctx, ino) if err != nil { logging.Debugf("Write inode %d: reserving extent-tree blocks failed: %v", f.inodeNum, err) - f.fs.freeUnused(ctx, f.inodeNum, ext.PhysicalBlock) - return nil, nil, nil, allocErrno(err) + giveBack() + return nil, nil, nil, false, allocErrno(err) } if err := f.fs.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { - f.fs.freeUnused(ctx, f.inodeNum, ext.PhysicalBlock) + giveBack() releaseTree() - return nil, nil, nil, syscall.EIO + return nil, nil, nil, false, syscall.EIO } var fresh *inode.Inode - fail := func(errno error) (*inode.Inode, *extent.Extent, func(), error) { + fail := func(errno error) (*inode.Inode, *extent.Extent, func(), bool, error) { f.fs.release(lock.LockTypeInode, uint64(f.inodeNum), "inode") if fresh != nil { f.fs.extentMgr.DiscardPending(ctx, fresh) } - f.fs.freeUnused(ctx, f.inodeNum, ext.PhysicalBlock) + giveBack() releaseTree() - return nil, nil, nil, errno + return nil, nil, nil, false, errno } fresh, err = f.fs.inodeTbl.Read(f.inodeNum) if err != nil { @@ -1947,7 +1998,7 @@ func (f *File) allocateBlockLocked(ctx context.Context, ino *inode.Inode, blockN } fresh.AllocatedBlocks += ext.Length fresh.WrittenBlocks += ext.Length - return fresh, ext, releaseTree, nil + return fresh, ext, releaseTree, reserved, nil } // writeInternal performs the actual write operation. @@ -1985,6 +2036,37 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i var cowDone []*extent.CoWResult // How many of allocatedExtents and cowDone a mid-write persist already put on disk. committedAllocs, committedCoW := 0, 0 + // No inode write may map these until their use is logged. + var reserveOut []uint64 + var intents []uint64 + defer func() { + for _, seq := range intents { + f.fs.closeIntent(seq) + } + }() + logReserveUse := func() error { + if len(reserveOut) == 0 { + return nil + } + var seq uint64 + err := f.fs.reserve.Logged(reserveOut, func() error { + txn := f.fs.journal.Begin() + if err := txn.LogBlockUse(reserveOut); err != nil { + f.fs.journal.Abort(txn) + return err + } + var err error + seq, err = f.fs.journal.Commit(txn) + return err + }) + if err != nil { + logging.Debugf("Write inode %d: logging the use of %d reserved blocks failed: %v", f.inodeNum, len(reserveOut), err) + return syscall.EIO + } + reserveOut = nil + intents = append(intents, seq) + return nil + } // An inode write that failed may still have reached the disk, so copies it // carried can no longer be freed. copiesMayBeOnDisk := false @@ -2009,6 +2091,11 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return nil } if inoDirty { + if err := logReserveUse(); err != nil { + releaseInodeLock() + f.fs.extentMgr.DiscardPending(ctx, ino) + return err + } copiesMayBeOnDisk = true if err := f.fs.inodeTbl.WriteWithLockHeld(ino); err != nil { logging.Debugf("Write inode %d: persisting inode before unlocking failed: %v", f.inodeNum, err) @@ -2029,6 +2116,9 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // Discarding frees tree blocks, which needs the bitmap lock: drop the inode lock first. releaseInodeLock() f.fs.extentMgr.DiscardPending(ctx, ino) + if len(reserveOut) > 0 { + f.fs.reserve.Untake(reserveOut...) + } if !copiesMayBeOnDisk { for _, cow := range cowDone[committedCoW:] { if err := f.fs.extentMgr.FreePhysicalBlock(ctx, cow.NewPhysBlock); err != nil { @@ -2105,17 +2195,24 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i } } - // The inode is re-read after allocation, so persist changes made under the lock first. - if err := persistAndUnlock(); err != nil { - return 0, err - } - - if f.fs.lockMgr != nil { - freshIno, newExt, releaseTree, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) + if newExt, ok := f.mapReadyBlockLocked(ctx, ino, blockNum, lockHeld); ok { + reserveOut = append(reserveOut, newExt.PhysicalBlock) + inoDirty = true + ext = newExt + newBlock = true + } else if f.fs.lockMgr != nil { + // The inode is re-read after allocation, so persist changes made under the lock first. + if err := persistAndUnlock(); err != nil { + return 0, err + } + freshIno, newExt, releaseTree, reserved, allocErr := f.allocateBlockLocked(ctx, ino, blockNum) if allocErr != nil { logging.Debugf("Write inode %d: allocating block %d failed: %v", f.inodeNum, blockNum, allocErr) return 0, allocErr } + if reserved { + reserveOut = append(reserveOut, newExt.PhysicalBlock) + } lockHeld = true releaseReservations = append(releaseReservations, releaseTree) ino = freshIno @@ -2478,7 +2575,18 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i // === COMMIT POINT: After this, operation is guaranteed recoverable === var commitErr error - intentSeq, commitErr = f.fs.journal.Commit(txn) + if len(reserveOut) > 0 { + commitErr = f.fs.reserve.Logged(reserveOut, func() error { + var err error + intentSeq, err = f.fs.journal.Commit(txn) + return err + }) + if commitErr == nil { + reserveOut = nil + } + } else { + intentSeq, commitErr = f.fs.journal.Commit(txn) + } if commitErr != nil { logging.Debugf("Write inode %d: journal commit failed: %v", f.inodeNum, commitErr) return 0, syscall.EIO diff --git a/internal/goclusterfs/reserve_power_cut_test.go b/internal/goclusterfs/reserve_power_cut_test.go new file mode 100644 index 00000000..35edb00c --- /dev/null +++ b/internal/goclusterfs/reserve_power_cut_test.go @@ -0,0 +1,144 @@ +package goclusterfs + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/reserve" +) + +// expectEmptyOr requires a file written once after it was created to be empty +// or to hold exactly what was written. +func expectEmptyOr(t *testing.T, f *File, data []byte) { + t.Helper() + if got := readWhole(t, f); len(got) != 0 && !bytes.Equal(got, data) { + t.Fatalf("inode %d: %d bytes that are neither empty nor what was written", f.inodeNum, len(got)) + } +} + +// expectNoLeaks requires that replay gave back every reserved block nothing used. +func expectNoLeaks(t *testing.T, fsys *FS) { + t.Helper() + r := runFsck(t, fsys, fsck.DefaultOptions()) + if strings.Contains(problemList(r), "leaked") { + t.Errorf("replay left reserved blocks leaked:%s", problemList(r)) + } +} + +// Power fails while a write takes the node's first reserve and maps a block from it. +func TestPowerCutDuringReserveRefill(t *testing.T) { + data := pattern(3, 4096) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + withReserve(t, fsys, 8, time.Hour) + f = createFile(t, fsys, "f") + }, + op: func(fsys *FS, _ *power) { + _, _ = f.Write(context.Background(), nil, data, 0) + }, + afterMount: expectNoLeaks, + check: func(t *testing.T, fsys *FS) { + expectEmptyOr(t, lookupFile(t, fsys, "f"), data) + }, + }) +} + +// Power fails while a write maps a block the node already holds in reserve. +func TestPowerCutDuringWriteFromReserve(t *testing.T) { + first, second := pattern(4, 4096), pattern(5, 4096) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + withReserve(t, fsys, 8, time.Hour) + writeAt(t, createFile(t, fsys, "first"), first, 0) + f = createFile(t, fsys, "second") + }, + op: func(fsys *FS, _ *power) { + _, _ = f.Write(context.Background(), nil, second, 0) + }, + afterMount: expectNoLeaks, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "first"), first) + expectEmptyOr(t, lookupFile(t, fsys, "second"), second) + }, + }) +} + +// Power fails during a write that maps two reserved blocks. The inode is +// written between the two, so the first block's use is logged before that. +func TestPowerCutDuringTwoBlockWriteFromReserve(t *testing.T) { + data := pattern(6, 8192) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + withReserve(t, fsys, 8, time.Hour) + f = createFile(t, fsys, "f") + }, + op: func(fsys *FS, _ *power) { + _, _ = f.Write(context.Background(), nil, data, BlockSize-4096) + }, + afterMount: expectNoLeaks, + check: func(t *testing.T, fsys *FS) { + got := readWhole(t, lookupFile(t, fsys, "f")) + want := append(make([]byte, BlockSize-4096), data...) + if len(got) != 0 && !bytes.Equal(got, want) { + t.Fatalf("%d bytes that are neither empty nor what was written", len(got)) + } + }, + }) +} + +// Power fails while the node gives its reserve back. Files keep their data, +// and no block is both free and mapped; a cut between ending the record and +// freeing the blocks leaks them, which repair reclaims. +func TestPowerCutDuringReserveReturn(t *testing.T) { + data := pattern(7, 4096) + var res *reserve.Manager + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + res = withReserve(t, fsys, 8, time.Hour) + writeAt(t, createFile(t, fsys, "f"), data, 0) + }, + op: func(fsys *FS, _ *power) { + res.ReturnAll(context.Background()) + }, + check: func(t *testing.T, fsys *FS) { + checkFile(t, lookupFile(t, fsys, "f"), data) + }, + }) +} + +// Power fails during a write that maps a reserved block into a hole and then +// copies a block a snapshot shares. The copy writes the inode first, so the +// reserved block's use is logged before that write. A cut between the two +// leaks the reserved block, which repair reclaims. +func TestPowerCutDuringReservedWriteThatCopies(t *testing.T) { + shared := pattern(8, 4096) + update := pattern(9, 8192) + off := int64(BlockSize - 4096) + var f *File + runPowerCuts(t, powerCutCase{ + setup: func(t *testing.T, fsys *FS) { + withReserve(t, fsys, 8, time.Hour) + f = createFile(t, fsys, "f") + writeAt(t, f, shared, BlockSize) + setxattr(t, f, "user.snapshot", "s1") + }, + op: func(fsys *FS, _ *power) { + _, _ = f.Write(context.Background(), nil, update, off) + }, + check: func(t *testing.T, fsys *FS) { + old := append(make([]byte, BlockSize), shared...) + updated := overwritten(old, int(off), update) + expectEachBlock(t, lookupFile(t, fsys, "f"), old, updated) + if !expectSnapshot(t, fsys, "s1", old) { + t.Fatal("snapshot s1 is gone") + } + }, + }) +} diff --git a/internal/goclusterfs/reserve_test.go b/internal/goclusterfs/reserve_test.go new file mode 100644 index 00000000..b6b392e0 --- /dev/null +++ b/internal/goclusterfs/reserve_test.go @@ -0,0 +1,264 @@ +package goclusterfs + +import ( + "context" + "fmt" + "sync" + "syscall" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/reserve" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +// withReserve gives fsys, which must have a journal, a block reserve as mount does. +func withReserve(t *testing.T, fsys *FS, size int, idle time.Duration) *reserve.Manager { + t.Helper() + res := reserve.New(reserve.Config{Size: size, Idle: idle}, fsys.bitmapMgr, fsys.journal, fsys.extentMgr.ReleaseBlocks, func() int { return 1 }) + fsys.bitmapMgr.SetNoSpaceWait(res) + fsys.journal.SetCheckpointHook(func() { res.Refresh(context.Background()) }) + fsys.SetReserve(res) + res.Start() + return res +} + +// journaledNode opens another node on path with its own journal slot. +func journaledNode(t *testing.T, path string, slot uint64) *FS { + t.Helper() + fsys := openFormattedFS(t, path) + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + size := sb.JournalSlotSize() + if fsys.journal, err = journal.Init(fsys.dev, sb.JournalRegionOffset+slot*size, size, fsys.nodeUUID, 1, journal.DefaultConfig()); err != nil { + t.Fatal(err) + } + return fsys +} + +// reserveFS opens a journaled node on an image large enough for 64-block +// reserves: below two reserves per node, or 1% of the blocks, there are none. +func reserveFS(t *testing.T) (fsys *FS, path string, offset, size uint64) { + t.Helper() + path = newFormattedImageOfSize(t, superblock.MinDeviceSize+1<<30) + fsys = journaledNode(t, path, 0) + if err := fsys.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + sb, err := superblock.Read(fsys.dev) + if err != nil { + t.Fatal(err) + } + return fsys, path, sb.JournalRegionOffset, sb.JournalSlotSize() +} + +func expectRefills(t *testing.T, res *reserve.Manager) { + t.Helper() + if res.Stats().Refills == 0 { + t.Fatal("the reserve never took blocks, so the test did not use it") + } +} + +func bitmapAcquires(fsys *FS) uint64 { + return lockAcquires(fsys)["bitmap"] +} + +// Writes take their blocks from the node's reserve, so ten one-block files +// take the bitmap lock once, and unmounting gives the rest back. +func TestWritesTakeBlocksFromReserve(t *testing.T) { + fsys, path, _, _ := reserveFS(t) + free := fsys.bitmapMgr.FreeCount() + withReserve(t, fsys, 64, time.Hour) + files := make([]*File, 10) + for i := range files { + files[i] = createFile(t, fsys, fmt.Sprintf("f%d", i)) + } + before := bitmapAcquires(fsys) + for i, f := range files { + writeAt(t, f, pattern(byte(i), 4096), 0) + } + if got := bitmapAcquires(fsys) - before; got != 1 { + t.Errorf("ten one-block writes took the bitmap lock %d times, want 1", got) + } + for i, f := range files { + checkFile(t, f, pattern(byte(i), 4096)) + } + if err := fsys.Close(); err != nil { + t.Fatal(err) + } + after := openFormattedFS(t, path) + if got := after.bitmapMgr.FreeCount(); got != free-10 { + t.Errorf("%d free blocks after unmount, want %d: the reserve was not given back", got, free-10) + } + expectCleanFsck(t, after) +} + +// A reserve nobody takes from goes back to the bitmap after the idle time. +func TestIdleReserveGoesBack(t *testing.T) { + fsys, _, _, _ := reserveFS(t) + free := fsys.bitmapMgr.FreeCount() + res := withReserve(t, fsys, 64, 200*time.Millisecond) + writeAt(t, createFile(t, fsys, "one"), pattern(1, 4096), 0) + deadline := time.Now().Add(10 * time.Second) + for res.Stats().Returns == 0 && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + if s := res.Stats(); s.Returns != 1 || s.Blocks != 0 { + t.Fatalf("reserve stats %+v after the idle time, want it given back once", s) + } + if got := fsys.bitmapMgr.FreeCount(); got != free-1 { + t.Errorf("%d free blocks after the idle return, want %d", got, free-1) + } +} + +// Node A takes block b from its reserve for a file, truncates the file, which +// frees b, and node B allocates b. A then stops without returning its reserve. +// Replaying A's journal frees A's unused reserved blocks and leaves B's block alone. +func TestReplayKeepsReservedBlockUsedFreedAndRetaken(t *testing.T) { + nodeA, path, offset, size := reserveFS(t) + resA := withReserve(t, nodeA, 8, time.Hour) + f := createFile(t, nodeA, "f") + writeAt(t, f, pattern(1, 4096), 0) + expectRefills(t, resA) + b := physicalBlock(t, nodeA, f.inodeNum, 0) + truncateFile(t, f, 0) + + nodeB := openFormattedFS(t, path) + got, err := nodeB.bitmapMgr.Alloc(context.Background(), 1) + if err != nil { + t.Fatal(err) + } + if got[0] != b { + t.Fatalf("node B got block %d, want the block %d node A used and freed", got[0], b) + } + nodeA.snapshotWkr.Stop() + nodeA.dev.SetWriteGate(func(int64) error { return errPowerCut }) + + nodeC := openFormattedFS(t, path) + replayCrashedJournal(t, nodeC, offset, size) + if !allocated(t, nodeC, b) { + t.Fatalf("replay freed block %d, which node B allocated after node A used and freed it", b) + } + // Node B allocated b without mapping it, so b is the one leak fsck may report. + r := runFsck(t, nodeC, fsck.DefaultOptions()) + for _, p := range r.Problems { + if p.Location != fmt.Sprintf("block %d", b) { + t.Errorf("after the replay fsck found more than node B's unmapped block %d:%s", b, problemList(r)) + break + } + } +} + +// Node B finds the bitmap full while node A holds blocks in reserve. B asks for +// them back and waits; A gives them back, and B's write gets a block instead of +// failing with ENOSPC. +func TestFullBitmapWaitsForPeersReserve(t *testing.T) { + path := newFormattedImageOfSize(t, superblock.MinDeviceSize+128<<20) + nodeA, nodeB := journaledNode(t, path, 0), journaledNode(t, path, 1) + if err := nodeA.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + resA := withReserve(t, nodeA, 4, time.Hour) + resB := withReserve(t, nodeB, 4, time.Hour) + defer expectRefills(t, resA) + resB.SetBroadcast(func() { go resA.ReturnAll(context.Background()) }) + writeAt(t, createFile(t, nodeA, "a"), pattern(1, 4096), 0) + if resA.Stats().Blocks == 0 { + t.Fatal("node A holds no reserve") + } + free := nodeB.bitmapMgr.FreeCount() + if _, err := nodeB.bitmapMgr.Alloc(context.Background(), int(free)); err != nil { + t.Fatalf("filling the bitmap: %v", err) + } + + start := time.Now() + n, errno := createFile(t, nodeB, "b").Write(context.Background(), nil, pattern(2, 4096), 0) + if errno != 0 || n != 4096 { + t.Fatalf("write with blocks in node A's reserve: n=%d errno=%v, want a block from the reserve", n, errno) + } + if s := resB.Stats(); s.Waits != 1 || time.Since(start) > reserve.MaxPeerWait { + t.Errorf("node B's waits: %+v after %v, want one wait shorter than %v", s, time.Since(start), reserve.MaxPeerWait) + } +} + +// With nothing left in any reserve, a write still fails with ENOSPC, after the +// wait for other nodes. +func TestFullBitmapWithoutReservesIsENOSPC(t *testing.T) { + if testing.Short() { + t.Skip("waits reserve.MaxPeerWait") + } + path := newFormattedImageOfSize(t, superblock.MinDeviceSize+128<<20) + fsys := journaledNode(t, path, 0) + if err := fsys.InitRoot(context.Background()); err != nil { + t.Fatal(err) + } + withReserve(t, fsys, 4, time.Hour) + f := createFile(t, fsys, "f") + if _, err := fsys.bitmapMgr.Alloc(context.Background(), int(fsys.bitmapMgr.FreeCount())); err != nil { + t.Fatal(err) + } + if _, errno := f.Write(context.Background(), nil, pattern(1, 4096), 0); errno != syscall.ENOSPC { + t.Fatalf("write on a full device: %v, want ENOSPC", errno) + } +} + +// Many writers share one reserve while it refills and goes back when idle. +// Every file keeps its data and fsck finds nothing after unmount. +func TestConcurrentWritersShareReserve(t *testing.T) { + fsys, path, _, _ := reserveFS(t) + res := withReserve(t, fsys, 8, 100*time.Millisecond) + const writers, perWriter = 4, 12 + files := make([][]*File, writers) + for w := range files { + for i := range perWriter { + files[w] = append(files[w], createFile(t, fsys, fmt.Sprintf("w%d-%d", w, i))) + } + } + var wg sync.WaitGroup + for w := range writers { + wg.Go(func() { + for i, f := range files[w] { + data := pattern(byte(w*perWriter+i), 2*4096) + if n, errno := f.Write(context.Background(), nil, data, int64(i%2)*BlockSize); errno != 0 || int(n) != len(data) { + t.Errorf("writer %d file %d: n=%d errno=%v", w, i, n, errno) + } + if i%5 == 0 { + time.Sleep(150 * time.Millisecond) + } + } + }) + } + wg.Wait() + expectRefills(t, res) + for w := range writers { + for i, f := range files[w] { + checkAt(t, f, pattern(byte(w*perWriter+i), 2*4096), int64(i%2)*BlockSize) + } + } + if err := fsys.Close(); err != nil { + t.Fatal(err) + } + expectCleanFsck(t, openFormattedFS(t, path)) +} + +// A reserve record stays pending while the reserve lasts, which keeps the +// journal head from passing it. Many small writes, as FUSE sends, fill the ring +// faster than the periodic checkpoint runs, so the record must be renewed as +// soon as the ring fills up, or commits fail with a full journal. +func TestReserveRecordDoesNotFillJournal(t *testing.T) { + fsys, _, _, _ := reserveFS(t) + withReserve(t, fsys, 64, time.Hour) + const files, writes, chunk = 100, 8, 128 << 10 + for i := range files { + f := createFile(t, fsys, fmt.Sprintf("f%d", i)) + for w := range writes { + if n, errno := f.Write(context.Background(), nil, pattern(byte(i), chunk), int64(w*chunk)); errno != 0 || n != chunk { + t.Fatalf("file %d write %d: n=%d errno=%v journal=%+v", i, w, n, errno, fsys.journal.Stats()) + } + } + } +} diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 13675f83..03ae3a3d 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -93,6 +93,8 @@ type Journal struct { activeTxns map[*Transaction]*ActiveTxn pendingIntents map[uint64]uint32 // committed but not yet completed, by seq, with where each starts in the ring + checkpointHook atomic.Pointer[func()] + hookRunning atomic.Bool // Checkpoint tracking lastCheckpointSeq uint64 @@ -225,9 +227,49 @@ func (j *Journal) checkpointLoop() { // periodicCheckpoint advances the head pointer for completed transactions. // A failed checkpoint is retried on the next tick. func (j *Journal) periodicCheckpoint() { + if hook := j.checkpointHook.Load(); hook != nil { + (*hook)() + } _, _ = j.Checkpoint() } +// SetCheckpointHook also runs fn early when the oldest intent is half a ring behind, since bursts fill the ring. +func (j *Journal) SetCheckpointHook(fn func()) { + j.checkpointHook.Store(&fn) +} + +func (j *Journal) runHookAsync() { + hook := j.checkpointHook.Load() + if hook == nil || !j.hookRunning.CompareAndSwap(false, true) { + return + } + go func() { + defer j.hookRunning.Store(false) + (*hook)() + }() +} + +func (j *Journal) oldestIntentLagLocked() uint32 { + size, tail := j.region.DataAreaSize(), j.region.TailOffset() + var oldest uint32 + for _, start := range j.pendingIntents { + oldest = max(oldest, (tail+size-start)%size) + } + return oldest +} + +// IntentLag reports in bytes how far intent seq starts behind the tail, and the ring size. +func (j *Journal) IntentLag(seq uint64) (lag, size uint32, ok bool) { + j.mu.Lock() + defer j.mu.Unlock() + start, ok := j.pendingIntents[seq] + if !ok { + return 0, 0, false + } + size = j.region.DataAreaSize() + return (j.region.TailOffset() + size - start) % size, size, true +} + // Checkpoint advances the head now instead of at the next interval. It reports false when the head could not move. func (j *Journal) Checkpoint() (bool, error) { j.mu.Lock() @@ -348,11 +390,15 @@ func (j *Journal) Commit(txn *Transaction) (uint64, error) { // The operation runs after Commit, so the head must not pass this transaction until CompleteIntent. delete(j.activeTxns, txn) j.pendingIntents[seq] = start + pressed := j.oldestIntentLagLocked() > j.region.DataAreaSize()/2 // Release lock BEFORE sync operations to avoid deadlock // The sync operations (CommitSync/CommitBatch) may need to acquire // other locks, so we must not hold j.mu during sync. j.mu.Unlock() + if pressed { + j.runHookAsync() + } // COMMIT POINT - sync to disk // After this barrier, the transaction is guaranteed durable. diff --git a/internal/reserve/reserve.go b/internal/reserve/reserve.go new file mode 100644 index 00000000..2ad1460d --- /dev/null +++ b/internal/reserve/reserve.go @@ -0,0 +1,377 @@ +// Package reserve hands writes data blocks from a per-node batch, recorded in the journal, without the bitmap lock. +package reserve + +import ( + "context" + "errors" + "log" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/supporttools/GoClusterFS/internal/journal" +) + +const ( + DefaultSize = 64 + DefaultIdle = 10 * time.Second + // MaxPeerWait bounds how long a full bitmap waits for peers' reserves before ENOSPC. + MaxPeerWait = 10 * time.Second +) + +type Bitmap interface { + Reserve(ctx context.Context, n int, record func([]uint64) error) ([]uint64, error) + FreeCount() uint64 + TotalBlocks() uint64 +} + +type Journal interface { + Begin() *journal.Transaction + Commit(txn *journal.Transaction) (uint64, error) + CompleteIntent(seq uint64) error + IntentLag(seq uint64) (lag, size uint32, ok bool) +} + +type Config struct { + Size int // blocks per reserve; 0 turns reserves off + Idle time.Duration // an untouched reserve goes back after this long +} + +type Stats struct { + Blocks int // blocks reserved and not handed out + Refills uint64 // reserves taken from the bitmap + Returns uint64 // reserves given back + Waits uint64 // allocations that found the bitmap full and waited for other nodes + WaitNanos uint64 // time those allocations waited +} + +type Manager struct { + cfg Config + bm Bitmap + j Journal + free func(ctx context.Context, blocks []uint64) + liveNodes func() int + broadcast func() + + // Shared while a write logs a use, exclusive while the record changes. + commits sync.RWMutex + mu sync.Mutex + untaken []uint64 + out map[uint64]bool // handed out, use not logged yet + seq uint64 // the reserve record, 0 when there is none + touched time.Time + + freeCount uint64 + countedAt time.Time + nodes int + nodesAt time.Time + + refills, returns, waits, waitNanos atomic.Uint64 + stop chan struct{} + done sync.WaitGroup +} + +func New(cfg Config, bm Bitmap, j Journal, free func(context.Context, []uint64), liveNodes func() int) *Manager { + if cfg.Idle <= 0 { + cfg.Idle = DefaultIdle + } + return &Manager{cfg: cfg, bm: bm, j: j, free: free, liveNodes: liveNodes, out: make(map[uint64]bool)} +} + +func (m *Manager) SetBroadcast(fn func()) { + m.broadcast = fn +} + +// Take reports false when reserves are off or space is low; the caller then uses the bitmap. +func (m *Manager) Take(ctx context.Context) (uint64, bool) { + if m.cfg.Size == 0 { + return 0, false + } + m.mu.Lock() + if len(m.untaken) > 0 && !m.lowSpaceLocked() { + defer m.mu.Unlock() + return m.handOutLocked(), true + } + m.mu.Unlock() + + m.commits.Lock() + defer m.commits.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if m.lowSpaceLocked() { + m.returnLocked(ctx) + return 0, false + } + if len(m.untaken) == 0 && !m.refillLocked(ctx) { + return 0, false + } + return m.handOutLocked(), true +} + +// TakeReady never refills, since refilling takes the bitmap lock and callers hold an inode lock. +func (m *Manager) TakeReady() (uint64, bool) { + if m.cfg.Size == 0 { + return 0, false + } + m.mu.Lock() + defer m.mu.Unlock() + if len(m.untaken) == 0 || m.lowSpaceLocked() { + return 0, false + } + return m.handOutLocked(), true +} + +func (m *Manager) handOutLocked() uint64 { + m.touched = time.Now() + b := m.untaken[0] + m.untaken = m.untaken[1:] + m.out[b] = true + return b +} + +func (m *Manager) Untake(blocks ...uint64) { + m.mu.Lock() + defer m.mu.Unlock() + for _, b := range blocks { + if m.out[b] { + delete(m.out, b) + m.untaken = append(m.untaken, b) + } + } + slices.Sort(m.untaken) +} + +// Logged runs log, which commits the use of blocks, while the reserve record cannot change. +func (m *Manager) Logged(blocks []uint64, log func() error) error { + m.commits.RLock() + defer m.commits.RUnlock() + if err := log(); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + for _, b := range blocks { + delete(m.out, b) + } + return nil +} + +// ReturnAll frees the untaken blocks; blocks still handed out move to a new record. +func (m *Manager) ReturnAll(ctx context.Context) { + m.commits.Lock() + defer m.commits.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + m.returnLocked(ctx) +} + +// Refresh re-logs a record that has fallen far behind the tail, so it stops pinning the journal head. +func (m *Manager) Refresh(ctx context.Context) { + m.commits.Lock() + defer m.commits.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if m.seq == 0 { + return + } + lag, size, ok := m.j.IntentLag(m.seq) + if !ok || lag < size/4 { + return + } + if err := m.relogLocked(append(slices.Clone(m.untaken), m.outBlocks()...)); err != nil { + log.Printf("[reserve] renewing the reserve record failed: %v", err) + } +} + +// WaitForPeers returns this node's reserve, asks peers for theirs and reports whether to retry. +func (m *Manager) WaitForPeers(ctx context.Context, waited time.Duration) bool { + if m.cfg.Size == 0 { + return false + } + if waited >= MaxPeerWait { + m.waits.Add(1) + m.waitNanos.Add(uint64(waited)) + log.Printf("[INFO] [reserve] no free blocks after waiting %v for other nodes' reserves", waited.Round(time.Millisecond)) + return false + } + if waited == 0 { + log.Printf("[INFO] [reserve] the bitmap is full; asking other nodes to give back their reserved blocks") + m.ReturnAll(ctx) + if m.broadcast != nil { + m.broadcast() + } + } + select { + case <-ctx.Done(): + return false + case <-time.After(200 * time.Millisecond): + } + return true +} + +func (m *Manager) Waited(d time.Duration) { + m.waits.Add(1) + m.waitNanos.Add(uint64(d)) + log.Printf("[INFO] [reserve] got blocks after waiting %v for other nodes' reserves", d.Round(time.Millisecond)) +} + +func (m *Manager) Start() { + if m.cfg.Size == 0 { + return + } + m.stop = make(chan struct{}) + m.done.Go(func() { + tick := time.NewTicker(m.cfg.Idle / 2) + defer tick.Stop() + for { + select { + case <-m.stop: + return + case <-tick.C: + m.mu.Lock() + idle := m.seq != 0 && len(m.out) == 0 && time.Since(m.touched) >= m.cfg.Idle + m.mu.Unlock() + if idle { + m.ReturnAll(context.Background()) + } + } + } + }) +} + +func (m *Manager) Stop(ctx context.Context) { + if m.stop != nil { + close(m.stop) + m.done.Wait() + m.stop = nil + } + m.ReturnAll(ctx) +} + +func (m *Manager) Stats() Stats { + m.mu.Lock() + n := len(m.untaken) + m.mu.Unlock() + return Stats{Blocks: n, Refills: m.refills.Load(), Returns: m.returns.Load(), Waits: m.waits.Load(), WaitNanos: m.waitNanos.Load()} +} + +func (m *Manager) lowSpaceLocked() bool { + free, nodes := m.freeCountLocked(), m.liveNodesLocked() + return free < max(m.bm.TotalBlocks()/100, uint64(2*m.cfg.Size*nodes)) +} + +// Blocks still handed out move into the new record. +func (m *Manager) refillLocked(ctx context.Context) bool { + n := min(m.cfg.Size, int(m.freeCountLocked()/uint64(16*m.liveNodesLocked()))) + if n <= 0 { + return false + } + carried := m.outBlocks() + var newSeq uint64 + blocks, err := m.bm.Reserve(ctx, n, func(picked []uint64) error { + var err error + newSeq, err = m.commitRecord(append(slices.Clone(picked), carried...)) + return err + }) + if err != nil { + if newSeq != 0 { + m.endLocked(newSeq, blocks) + } + if !errors.Is(err, context.Canceled) { + log.Printf("[reserve] taking %d blocks failed: %v", n, err) + } + return false + } + if m.seq != 0 { + if err := m.j.CompleteIntent(m.seq); err != nil { + log.Printf("[reserve] ending reserve record %d failed: %v", m.seq, err) + } + } + m.seq, m.untaken = newSeq, blocks + m.freeCount -= uint64(len(blocks)) + m.refills.Add(1) + return true +} + +// Ending the record before freeing means a crash leaks blocks instead of replay double-freeing them. +func (m *Manager) returnLocked(ctx context.Context) { + if m.seq == 0 { + return + } + untaken := m.untaken + m.untaken = nil + if out := m.outBlocks(); len(out) > 0 { + if err := m.relogLocked(out); err != nil { + log.Printf("[reserve] keeping %d reserved blocks: renewing the record failed: %v", len(untaken), err) + m.untaken = untaken + return + } + } else { + if err := m.j.CompleteIntent(m.seq); err != nil { + log.Printf("[reserve] keeping %d reserved blocks: ending the record failed: %v", len(untaken), err) + m.untaken = untaken + return + } + m.seq = 0 + } + if len(untaken) > 0 { + m.free(ctx, untaken) + m.freeCount += uint64(len(untaken)) + } + m.returns.Add(1) +} + +func (m *Manager) relogLocked(blocks []uint64) error { + seq, err := m.commitRecord(blocks) + if err != nil { + return err + } + if err := m.j.CompleteIntent(m.seq); err != nil { + log.Printf("[reserve] ending reserve record %d failed: %v", m.seq, err) + } + m.seq = seq + return nil +} + +func (m *Manager) endLocked(seq uint64, blocks []uint64) { + if err := m.j.CompleteIntent(seq); err != nil { + log.Printf("[reserve] leaving %d blocks allocated: ending record %d failed: %v", len(blocks), seq, err) + return + } + m.free(context.Background(), blocks) +} + +func (m *Manager) commitRecord(blocks []uint64) (uint64, error) { + txn := m.j.Begin() + if err := txn.LogBlockReserve(blocks); err != nil { + return 0, err + } + return m.j.Commit(txn) +} + +func (m *Manager) outBlocks() []uint64 { + out := make([]uint64, 0, len(m.out)) + for b := range m.out { + out = append(out, b) + } + slices.Sort(out) + return out +} + +func (m *Manager) freeCountLocked() uint64 { + if time.Since(m.countedAt) > 2*time.Second { + m.freeCount, m.countedAt = m.bm.FreeCount(), time.Now() + } + return m.freeCount +} + +func (m *Manager) liveNodesLocked() int { + if m.liveNodes == nil { + return 1 + } + if time.Since(m.nodesAt) > 10*time.Second { + m.nodes, m.nodesAt = max(1, m.liveNodes()), time.Now() + } + return m.nodes +} diff --git a/internal/reserve/reserve_test.go b/internal/reserve/reserve_test.go new file mode 100644 index 00000000..81eb5306 --- /dev/null +++ b/internal/reserve/reserve_test.go @@ -0,0 +1,178 @@ +package reserve + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/journal" +) + +type fakeBitmap struct { + total uint64 + free []uint64 +} + +func newFakeBitmap(total, used uint64) *fakeBitmap { + b := &fakeBitmap{total: total} + for i := used; i < total; i++ { + b.free = append(b.free, i) + } + return b +} + +func (b *fakeBitmap) Reserve(_ context.Context, n int, record func([]uint64) error) ([]uint64, error) { + n = min(n, len(b.free)) + picked := slices.Clone(b.free[:n]) + if err := record(picked); err != nil { + return nil, err + } + b.free = b.free[n:] + return picked, nil +} + +func (b *fakeBitmap) FreeCount() uint64 { return uint64(len(b.free)) } +func (b *fakeBitmap) TotalBlocks() uint64 { return b.total } +func (b *fakeBitmap) give(_ context.Context, blocks []uint64) { + b.free = append(b.free, blocks...) + slices.Sort(b.free) +} + +// fakeJournal records the order of commits and completions. +type fakeJournal struct { + seq uint64 + events []string + completed map[uint64]bool + lag uint32 + failNext bool +} + +func (j *fakeJournal) Begin() *journal.Transaction { return journal.NewTransaction(0, [16]byte{1}, 1) } +func (j *fakeJournal) Commit(*journal.Transaction) (uint64, error) { + if j.failNext { + j.failNext = false + return 0, errors.New("commit failed") + } + j.seq++ + j.events = append(j.events, "commit") + return j.seq, nil +} +func (j *fakeJournal) CompleteIntent(seq uint64) error { + if j.completed == nil { + j.completed = map[uint64]bool{} + } + j.completed[seq] = true + j.events = append(j.events, "complete") + return nil +} +func (j *fakeJournal) IntentLag(seq uint64) (uint32, uint32, bool) { + return j.lag, 1000, !j.completed[seq] && seq != 0 +} + +func newManager(size int, bm *fakeBitmap, j *fakeJournal) *Manager { + return New(Config{Size: size, Idle: time.Hour}, bm, j, func(ctx context.Context, blocks []uint64) { + j.events = append(j.events, "free") + bm.give(ctx, blocks) + }, func() int { return 1 }) +} + +// A reserve holds no more than the configured size, nor a sixteenth of the free +// blocks per node, and none at all below two reserves per node or 1% of the blocks. +func TestReserveSize(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + total, used uint64 + size int + want int + }{ + {"configured size", 10000, 0, 64, 64}, + {"sixteenth of free", 10000, 9000, 128, 62}, + {"below two reserves", 10000, 9900, 64, 0}, + {"below 1%", 100000, 99050, 8, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + bm := newFakeBitmap(tc.total, tc.used) + m := newManager(tc.size, bm, &fakeJournal{}) + _, ok := m.Take(ctx) + got := 0 + if ok { + got = m.Stats().Blocks + 1 + } + if got != tc.want { + t.Errorf("reserve of %d blocks, want %d", got, tc.want) + } + }) + } +} + +// Giving the reserve back ends its record before it frees anything, so a crash +// in between leaks blocks instead of letting replay free them twice. +func TestReturnEndsRecordBeforeFreeing(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + b, _ := m.Take(context.Background()) + if err := m.Logged([]uint64{b}, func() error { return nil }); err != nil { + t.Fatal(err) + } + j.events = nil + m.ReturnAll(context.Background()) + if want := []string{"complete", "free"}; !slices.Equal(j.events, want) { + t.Errorf("return did %v, want %v", j.events, want) + } + if got := bm.FreeCount(); got != 10000-1 { + t.Errorf("%d free blocks after the return, want %d", got, 10000-1) + } +} + +// A block a write took and has not logged keeps a record: returning the +// reserve writes a new record for it before ending the old one and freeing the rest. +func TestReturnKeepsHandedOutBlocksRecorded(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + b, _ := m.Take(context.Background()) + j.events = nil + m.ReturnAll(context.Background()) + if want := []string{"commit", "complete", "free"}; !slices.Equal(j.events, want) { + t.Errorf("return did %v, want %v", j.events, want) + } + m.Untake(b) + j.events = nil + m.ReturnAll(context.Background()) + if want := []string{"complete", "free"}; !slices.Equal(j.events, want) || bm.FreeCount() != 10000 { + t.Errorf("second return did %v with %d free, want %v with all free", j.events, bm.FreeCount(), []string{"complete", "free"}) + } +} + +// A write whose commit fails keeps its blocks in the reserve to put back. +func TestFailedLogKeepsBlocksOut(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + b, _ := m.Take(context.Background()) + if err := m.Logged([]uint64{b}, func() error { return errors.New("commit failed") }); err == nil { + t.Fatal("Logged hid the commit error") + } + m.Untake(b) + if got := m.Stats().Blocks; got != 8 { + t.Errorf("%d blocks in reserve after putting the block back, want 8", got) + } +} + +// The record is written again once it is a quarter of the journal behind the tail. +func TestRefreshRenewsAnOldRecord(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + m.Take(context.Background()) + j.events, j.lag = nil, 100 + m.Refresh(context.Background()) + if len(j.events) != 0 { + t.Errorf("refresh of a recent record did %v, want nothing", j.events) + } + j.lag = 300 + m.Refresh(context.Background()) + if want := []string{"commit", "complete"}; !slices.Equal(j.events, want) { + t.Errorf("refresh of an old record did %v, want %v", j.events, want) + } +} diff --git a/internal/stats/sampler.go b/internal/stats/sampler.go index a8925cc5..a4f668df 100644 --- a/internal/stats/sampler.go +++ b/internal/stats/sampler.go @@ -11,6 +11,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/journal" "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/reserve" "github.com/supporttools/GoClusterFS/internal/trim" ) @@ -23,6 +24,9 @@ type CacheSource interface{ Stats() cache.Stats } type LockSource interface{ Stats() lock.Stats } +// ReserveSource reports the node's block reserve. +type ReserveSource interface{ Stats() reserve.Stats } + type DirSource interface{ IndexStats() dir.IndexStats } type JournalSource interface{ Stats() journal.JournalStats } @@ -50,6 +54,7 @@ type Sources struct { Trim TrimSource Blocks BlockSource Nodes NodeSource + Reserve ReserveSource BlockSize uint64 } @@ -144,6 +149,11 @@ func (s *Sampler) sampleCounters() { if s.src.Journal != nil { s.current.Journal = journalResponse(s.src.Journal.Stats()) } + if s.src.Reserve != nil { + r := s.src.Reserve.Stats() + s.current.Reserve = &api.ReserveStatsResponse{Blocks: r.Blocks, Refills: r.Refills, Returns: r.Returns, + PeerWaits: r.Waits, PeerWaitSeconds: time.Duration(r.WaitNanos).Seconds()} + } s.current.Trim = &api.TrimStatsResponse{} if s.src.Trim != nil { s.current.Trim = trimResponse(s.src.Trim.Stats()) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 0311e01b..8dbc48e5 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3106,6 +3106,127 @@ scenarios: environments: go: internal/goclusterfs/reserve_crash_test.go::TestFsckRepairFreesCrashedNodesReserveOnce + - id: reserve-size + name: Reserve size and low-space cutoff + description: A reserve holds at most the configured size and a sixteenth of the free blocks per node, and none below two reserves per node or 1% of the blocks + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestReserveSize + + - id: reserve-return-order + name: Reserve return ends its record first + description: Giving the reserve back ends its journal record before freeing blocks, so a crash in between leaks rather than double-frees + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestReturnEndsRecordBeforeFreeing + + - id: reserve-return-handed-out + name: Returned reserve keeps handed-out blocks recorded + description: Blocks a write took and has not logged stay in a new record when the reserve is given back, and go back later + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestReturnKeepsHandedOutBlocksRecorded + + - id: reserve-failed-log + name: Failed commit keeps reserved blocks + description: A write whose commit fails keeps its reserved blocks to put back + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestFailedLogKeepsBlocksOut + + - id: reserve-refresh + name: Old reserve record is renewed + description: The reserve record is written again once it is a quarter of the journal behind the tail + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestRefreshRenewsAnOldRecord + + - id: reserve-writes + name: Writes take blocks from the reserve + description: Ten one-block writes take the bitmap lock once, and unmount gives the rest of the reserve back; fsck is clean + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestWritesTakeBlocksFromReserve + + - id: reserve-idle-return + name: Idle reserve goes back + description: A reserve nobody takes from goes back to the bitmap after the idle time + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestIdleReserveGoesBack + + - id: reserve-replay-used-freed-retaken + name: Replay keeps a reserved block used, freed and retaken + description: Node A writes a reserved block, truncates it and node B allocates it; replaying A's journal frees A's unused reserve and leaves B's block alone + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestReplayKeepsReservedBlockUsedFreedAndRetaken + + - id: reserve-full-waits-for-peers + name: Full bitmap waits for peers' reserves + description: Node B finds the bitmap full while node A holds a reserve; B asks for it back and its write gets a block instead of ENOSPC + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestFullBitmapWaitsForPeersReserve + + - id: reserve-full-enospc + name: Full bitmap without reserves is ENOSPC + description: With no blocks in any reserve, a write on a full device fails with ENOSPC after the wait for other nodes + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestFullBitmapWithoutReservesIsENOSPC + + - id: reserve-concurrent-writers + name: Concurrent writers share a reserve + description: Four writers share one reserve while it refills and goes back when idle; every file keeps its data and fsck is clean + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestConcurrentWritersShareReserve + + - id: power-cut-reserve-refill + name: Power cut during a reserve refill + description: Power fails at every write of a write that takes the first reserve and maps a block; replay leaves no leaked blocks + priority: go_only + environments: + go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringReserveRefill + + - id: power-cut-write-from-reserve + name: Power cut during a write from the reserve + description: Power fails at every write of a write that maps a block the reserve holds; replay leaves no leaked blocks + priority: go_only + environments: + go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringWriteFromReserve + + - id: power-cut-two-block-reserve + name: Power cut during a two-block reserved write + description: Power fails at every write of a write that maps two reserved blocks; replay leaves no leaked blocks + priority: go_only + environments: + go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringTwoBlockWriteFromReserve + + - id: power-cut-reserve-return + name: Power cut during a reserve return + description: Power fails at every write of giving the reserve back; files keep their data and fsck finds nothing but leaks + priority: go_only + environments: + go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringReserveReturn + + - id: power-cut-reserved-write-copies + name: Power cut during a reserved write that copies + description: Power fails at every write of a write that maps a reserved block and copies a shared one; the file and its snapshot keep their data + priority: go_only + environments: + go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringReservedWriteThatCopies + + - id: reserve-returned-after-crash + name: Killed node's reserve is returned + description: A node takes a block reserve, writes one block from it and is killed; the survivor replays its journal and the unused reserved blocks come back, so free space ends one block below where it started + priority: high + environments: + go: internal/goclusterfs/reserve_test.go::TestReplayKeepsReservedBlockUsedFreedAndRetaken + docker: TODO - Docker kills containers but the harness has no df comparison across nodes; the Go replay test and the VM test cover it + vm: run-vm-tests.sh::test_reserve_returned_after_crash + # =========================================================================== # DISTRIBUTED LOCK SAFETY (internal/lock) # Two or three lock managers share one device to act as separate nodes. diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index e0ff120d..4d67c2b1 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -586,6 +586,61 @@ test_journal_recovery_multinode() { return 0 } +# VM A takes a block reserve, writes one block from it and is killed. VM B +# replays VM A's journal, which gives the unused reserved blocks back: VM B's +# free space ends one block below where it started. +test_reserve_returned_after_crash() { + log_info "Testing that a killed node's unused block reserve is returned..." + + if [[ $NODE_COUNT -lt 2 ]]; then + log_info "SKIP: Need at least 2 VMs" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + + local vm_a=1 vm_b=2 + local file=/mnt/goclusterfs/reserve-crash.bin + local free_kb + free_kb() { ssh_run "$vm_b" "df -k /mnt/goclusterfs | awk 'NR==2 {print \$4}'"; } + + local before + before=$(free_kb) + ssh_run "$vm_a" "dd if=/dev/urandom of=$file bs=1M count=1 conv=fsync 2>/dev/null" || return 1 + local reserved + reserved=$(free_kb) + log_info "free on VM B: ${before}KB before, ${reserved}KB while VM A holds its reserve" + if (( reserved > before - 2048 )); then + log_error "VM A's write took less than its block and a reserve (${before}KB -> ${reserved}KB)" + return 1 + fi + + log_info "Hard-killing VM A while it holds its reserve..." + kill_vm "$vm_a" + local want=$((before - 1024)) now=0 waited=0 + while (( waited < 90 )); do + now=$(free_kb) + [[ "$now" -eq "$want" ]] && break + sleep 3 + waited=$((waited + 3)) + done + if [[ "$now" -ne "$want" ]]; then + log_error "VM B shows ${now}KB free after VM A's replay, want ${want}KB: the reserve was not returned" + return 1 + fi + if [[ "$(ssh_run "$vm_b" "stat -c %s $file")" != "1048576" ]]; then + log_error "VM A's file is not whole on VM B after the replay" + return 1 + fi + log_info "VM A's unused reserve came back after ${waited}s" + + start_vm "$vm_a" false + wait_vm_healthy "$vm_a" + setup_goclusterfs "$vm_a" false + ssh_run "$vm_b" "rm -f $file" + log_info "Reserve returned after crash test passed" + return 0 +} + # VM A's mount restarts while VM B keeps creating files. # Matches Go test: TestMultiNodeRestartKeepsPeerJournal in journal_slots_test.go test_restart_keeps_peer_journal() { @@ -3896,6 +3951,7 @@ main() { run_test "Concurrent File Creation" test_concurrent_file_creation run_test "VM Kill Recovery" test_vm_kill_recovery run_test "Journal Recovery Multi-Node" test_journal_recovery_multinode + run_test "Reserve Returned After Crash" test_reserve_returned_after_crash run_test "Restart Keeps Peer Journal" test_restart_keeps_peer_journal run_test "Journal Recovery Multiple Files" test_journal_recovery_multiple_files run_test "Journal Recovery Large File" test_journal_recovery_large_file From d00a19f2416b044462a6fed26704b9bc2cb4195a Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 00:30:23 -0600 Subject: [PATCH 2/7] test(reserve): check a crashed node's written reserve blocks survive replay Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/reserve_test.go | 40 ++++++++++++++++++++++++++++ tests/test-manifest.yaml | 6 ++--- tests/vm/scripts/run-vm-tests.sh | 18 ++++++++++--- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/internal/goclusterfs/reserve_test.go b/internal/goclusterfs/reserve_test.go index b6b392e0..b2de37a3 100644 --- a/internal/goclusterfs/reserve_test.go +++ b/internal/goclusterfs/reserve_test.go @@ -153,6 +153,46 @@ func TestReplayKeepsReservedBlockUsedFreedAndRetaken(t *testing.T) { } } +// Node A writes two files from its reserve, the second through two blocks, and +// crashes holding the rest. After replay the files keep their data and no +// allocation can take their blocks. +func TestReplayKeepsMappedBlocksOfCrashedNodesReserve(t *testing.T) { + nodeA, path, offset, size := reserveFS(t) + fOne, fTwo := createFile(t, nodeA, "one"), createFile(t, nodeA, "two") + free := nodeA.bitmapMgr.FreeCount() + resA := withReserve(t, nodeA, 8, time.Hour) + one, two := pattern(1, 4096), pattern(2, 8192) + writeAt(t, fOne, one, 0) + writeAt(t, fTwo, two, BlockSize-4096) + expectRefills(t, resA) + nodeA.snapshotWkr.Stop() + nodeA.dev.SetWriteGate(func(int64) error { return errPowerCut }) + + nodeB := openFormattedFS(t, path) + replayCrashedJournal(t, nodeB, offset, size) + fOne, fTwo = lookupFile(t, nodeB, "one"), lookupFile(t, nodeB, "two") + checkFile(t, fOne, one) + checkFile(t, fTwo, append(make([]byte, BlockSize-4096), two...)) + mapped := map[uint64]bool{ + physicalBlock(t, nodeB, fOne.inodeNum, 0): true, + physicalBlock(t, nodeB, fTwo.inodeNum, 0): true, + physicalBlock(t, nodeB, fTwo.inodeNum, 1): true, + } + if got, want := nodeB.bitmapMgr.FreeCount(), free-uint64(len(mapped)); got != want { + t.Errorf("%d free blocks after the replay, want %d: only the %d mapped blocks stay allocated", got, want, len(mapped)) + } + expectCleanFsck(t, nodeB) + rest, err := nodeB.bitmapMgr.Alloc(context.Background(), int(nodeB.bitmapMgr.FreeCount())) + if err != nil { + t.Fatal(err) + } + for _, b := range rest { + if mapped[b] { + t.Fatalf("node B allocated block %d, which node A's file maps", b) + } + } +} + // Node B finds the bitmap full while node A holds blocks in reserve. B asks for // them back and waits; A gives them back, and B's write gets a block instead of // failing with ENOSPC. diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 8dbc48e5..65b44954 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3219,11 +3219,11 @@ scenarios: go: internal/goclusterfs/reserve_power_cut_test.go::TestPowerCutDuringReservedWriteThatCopies - id: reserve-returned-after-crash - name: Killed node's reserve is returned - description: A node takes a block reserve, writes one block from it and is killed; the survivor replays its journal and the unused reserved blocks come back, so free space ends one block below where it started + name: Killed node's reserve is returned and its written blocks kept + description: A node writes from its block reserve and is killed; the survivor replays its journal, the unused reserved blocks come back, and the written file keeps its data and blocks through later allocations priority: high environments: - go: internal/goclusterfs/reserve_test.go::TestReplayKeepsReservedBlockUsedFreedAndRetaken + go: internal/goclusterfs/reserve_test.go::TestReplayKeepsMappedBlocksOfCrashedNodesReserve docker: TODO - Docker kills containers but the harness has no df comparison across nodes; the Go replay test and the VM test cover it vm: run-vm-tests.sh::test_reserve_returned_after_crash diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 4d67c2b1..163317e0 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -606,6 +606,8 @@ test_reserve_returned_after_crash() { local before before=$(free_kb) ssh_run "$vm_a" "dd if=/dev/urandom of=$file bs=1M count=1 conv=fsync 2>/dev/null" || return 1 + local sum + sum=$(ssh_run "$vm_a" "md5sum $file | cut -d' ' -f1") local reserved reserved=$(free_kb) log_info "free on VM B: ${before}KB before, ${reserved}KB while VM A holds its reserve" @@ -627,16 +629,24 @@ test_reserve_returned_after_crash() { log_error "VM B shows ${now}KB free after VM A's replay, want ${want}KB: the reserve was not returned" return 1 fi - if [[ "$(ssh_run "$vm_b" "stat -c %s $file")" != "1048576" ]]; then - log_error "VM A's file is not whole on VM B after the replay" + log_info "VM A's unused reserve came back after ${waited}s" + + # Allocation is first-fit, so VM B's filler would take the file's block if replay had freed it. + local filler=/mnt/goclusterfs/reserve-crash-filler.bin + ssh_run "$vm_b" "dd if=/dev/zero of=$filler bs=1M count=128 conv=fsync 2>/dev/null" || return 1 + if [[ "$(ssh_run "$vm_b" "md5sum $file | cut -d' ' -f1")" != "$sum" ]]; then + log_error "VM A's file changed on VM B after the replay and a 128MB write" return 1 fi - log_info "VM A's unused reserve came back after ${waited}s" start_vm "$vm_a" false wait_vm_healthy "$vm_a" setup_goclusterfs "$vm_a" false - ssh_run "$vm_b" "rm -f $file" + if [[ "$(ssh_run "$vm_a" "md5sum $file | cut -d' ' -f1")" != "$sum" ]]; then + log_error "VM A reads different data from its file after restarting" + return 1 + fi + ssh_run "$vm_b" "rm -f $file $filler" log_info "Reserve returned after crash test passed" return 0 } From e596c5296a51c245f31ab249d1e3c2173228f987 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 01:03:13 -0600 Subject: [PATCH 3/7] fix(statfs): count the node's own unused reserve as free A write that refilled the reserve lowered df by the whole reserve, and an unmount raised it again, so the statfs tests saw free space jump. df now shows what this node can write without waiting. Other nodes' reserves still count as used until they go back, so the shell tests that compare free space wait out the idle return first. Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 2 +- internal/goclusterfs/fs.go | 3 +++ internal/goclusterfs/reserve_test.go | 22 ++++++++++++++++++++++ internal/stats/sampler.go | 6 +++++- tests/docker/scripts/run-tests.sh | 7 +++++++ tests/vm/scripts/run-vm-tests.sh | 8 ++++++++ 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index c7393759..57fde59a 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -313,7 +313,7 @@ Each node takes up to 64 free blocks from the block bitmap at once (`-block-rese - **Size.** A reserve holds at most `-block-reserve` blocks, and at most 1/16 of the free blocks per mounted node. There are no reserves while fewer blocks are free than 1% of the device or two reserves per mounted node. - **Going back.** A node gives its unused reserve back after 10 seconds without a write, at a clean unmount, and when a write finds free space below that line. - **Full device.** A node that finds no free block gives back its own reserve, asks the other nodes over broadcast to give back theirs, and retries for up to 10 seconds before it fails with `ENOSPC`. It logs the wait at INFO, and `goclusterfs_reserve_peer_waits_total` and `goclusterfs_reserve_peer_wait_seconds_total` count it. Without broadcast, other nodes still give back idle reserves within 10 seconds. -- **`df`.** Reserved blocks count as used until a write uses them or the node gives them back: at most 64 MB per node at the default. +- **`df`.** A node counts its own unused reserve as free, since its writes can use it. Other nodes' reserves count as used until they give them back, so `df` on two nodes can differ by up to 64 MB per node at the default. - **Crash.** Each reserve is a journal record. When the node's journal is replayed, by the node at its next mount or by another node, the blocks it reserved and never used go back to the bitmap. fsck reports them as "reserved by node N and never used", which is not an error, and `fsck -repair -force` frees them after it marks the reserve done in that node's journal, so a later replay does not free them again. A crash in a few narrow windows, such as between ending a reserve and freeing it, leaks a few blocks instead; `fsck -repair` reclaims them. ### Handling Node Failure diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index a535805e..d8465752 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -596,6 +596,9 @@ func (f *FS) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno { totalBlocks := f.bitmapMgr.TotalBlocks() freeBlocks := f.bitmapMgr.FreeCount() + if f.reserve != nil { + freeBlocks += uint64(f.reserve.Stats().Blocks) + } out.Blocks = totalBlocks out.Bfree = freeBlocks diff --git a/internal/goclusterfs/reserve_test.go b/internal/goclusterfs/reserve_test.go index b2de37a3..306c046c 100644 --- a/internal/goclusterfs/reserve_test.go +++ b/internal/goclusterfs/reserve_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/hanwen/go-fuse/v2/fuse" + "github.com/supporttools/GoClusterFS/internal/fsck" "github.com/supporttools/GoClusterFS/internal/journal" "github.com/supporttools/GoClusterFS/internal/reserve" @@ -97,6 +99,26 @@ func TestWritesTakeBlocksFromReserve(t *testing.T) { expectCleanFsck(t, after) } +// statfs counts the node's unused reserve as free, so a one-block write lowers +// it by one block even though the write took a whole reserve from the bitmap. +func TestStatfsCountsOwnReserveAsFree(t *testing.T) { + fsys, _, _, _ := reserveFS(t) + f := createFile(t, fsys, "one") + var before, after fuse.StatfsOut + if errno := fsys.Statfs(context.Background(), &before); errno != 0 { + t.Fatal(errno) + } + res := withReserve(t, fsys, 64, time.Hour) + writeAt(t, f, pattern(1, 4096), 0) + expectRefills(t, res) + if errno := fsys.Statfs(context.Background(), &after); errno != 0 { + t.Fatal(errno) + } + if after.Bfree != before.Bfree-1 || after.Bavail != after.Bfree { + t.Errorf("statfs Bfree %d, Bavail %d after a one-block write, want both %d", after.Bfree, after.Bavail, before.Bfree-1) + } +} + // A reserve nobody takes from goes back to the bitmap after the idle time. func TestIdleReserveGoesBack(t *testing.T) { fsys, _, _, _ := reserveFS(t) diff --git a/internal/stats/sampler.go b/internal/stats/sampler.go index a4f668df..7080980d 100644 --- a/internal/stats/sampler.go +++ b/internal/stats/sampler.go @@ -167,7 +167,11 @@ func (s *Sampler) sampleCounters() { func (s *Sampler) sampleDevice() { s.lastDevice = time.Now() if s.src.Blocks != nil { - s.capacity.Store(capacityInfo(s.src.Blocks.TotalBlocks(), s.src.Blocks.FreeCount(), s.src.BlockSize)) + free := s.src.Blocks.FreeCount() + if s.src.Reserve != nil { + free += uint64(s.src.Reserve.Stats().Blocks) + } + s.capacity.Store(capacityInfo(s.src.Blocks.TotalBlocks(), free, s.src.BlockSize)) } if s.src.Nodes != nil { slots, _ := s.src.Nodes.Slots() diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index c2d932d7..ad382f51 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2471,6 +2471,9 @@ statfs_free_blocks() { exec_on_node "$1" stat -f -c %f /mnt/goclusterfs } +# statfs counts other nodes' unused block reserves as used; they go back within 1.5 x the 10s idle time. +RESERVE_IDLE_RETURN_SECS=16 + # Writes 5 blocks, restarts the node, and checks that statfs reports the same # free blocks as before the restart, not the whole volume. test_statfs_after_remount() { @@ -2481,6 +2484,7 @@ test_statfs_after_remount() { exec_on_node "$node" rm -f "$file" local before written remounted + sleep "$RESERVE_IDLE_RETURN_SECS" before=$(statfs_free_blocks "$node") exec_on_node "$node" dd if=/dev/urandom of="$file" bs=1M count=5 2>/dev/null written=$(statfs_free_blocks "$node") @@ -2518,8 +2522,10 @@ test_statfs_multinode() { exec_on_node "$writer" rm -f "$file" local start on_writer on_remover + sleep "$RESERVE_IDLE_RETURN_SECS" start=$(statfs_free_blocks "$remover") exec_on_node "$writer" dd if=/dev/urandom of="$file" bs=1M count=5 2>/dev/null + sleep "$RESERVE_IDLE_RETURN_SECS" on_writer=$(statfs_free_blocks "$writer") on_remover=$(statfs_free_blocks "$remover") if [[ "$on_writer" != "$on_remover" ]] || (( start - on_remover < 5 )); then @@ -3114,6 +3120,7 @@ test_snapshot_delete_after_overwrite() { exec_on_node "$node" rm -f "$file" exec_on_node "$node" sh -c "head -c 2097152 /dev/urandom > $file" || return 1 + sleep "$RESERVE_IDLE_RETURN_SECS" free_before=$(exec_on_node "$node" stat -f -c %f /mnt/goclusterfs) exec_on_node "$node" setfattr -n user.snapshot -v snap_delete_temp "$file" || { log "ERROR: snapshot failed" diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 163317e0..de050a37 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -604,6 +604,7 @@ test_reserve_returned_after_crash() { free_kb() { ssh_run "$vm_b" "df -k /mnt/goclusterfs | awk 'NR==2 {print \$4}'"; } local before + sleep "$RESERVE_IDLE_RETURN_SECS" before=$(free_kb) ssh_run "$vm_a" "dd if=/dev/urandom of=$file bs=1M count=1 conv=fsync 2>/dev/null" || return 1 local sum @@ -1917,6 +1918,9 @@ statfs_free_blocks_on_vm() { ssh_run "$1" "stat -f -c %f /mnt/goclusterfs" } +# statfs counts other nodes' unused block reserves as used; they go back within 1.5 x the 10s idle time. +RESERVE_IDLE_RETURN_SECS=16 + # Writes 5 blocks, remounts on a restarted VM, and checks that statfs reports # the same free blocks as before the restart, not the whole volume. test_statfs_after_remount() { @@ -1925,6 +1929,7 @@ test_statfs_after_remount() { local file=/mnt/goclusterfs/statfs_remount.bin local before written remounted + sleep "$RESERVE_IDLE_RETURN_SECS" before=$(statfs_free_blocks_on_vm 1) ssh_run 1 "dd if=/dev/urandom of=$file bs=1M count=5 2>/dev/null" written=$(statfs_free_blocks_on_vm 1) @@ -1959,8 +1964,10 @@ test_statfs_multinode() { local file=/mnt/goclusterfs/statfs_multinode.bin local start on_writer on_remover + sleep "$RESERVE_IDLE_RETURN_SECS" start=$(statfs_free_blocks_on_vm 2) ssh_run 1 "dd if=/dev/urandom of=$file bs=1M count=5 2>/dev/null" + sleep "$RESERVE_IDLE_RETURN_SECS" on_writer=$(statfs_free_blocks_on_vm 1) on_remover=$(statfs_free_blocks_on_vm 2) if [[ "$on_writer" != "$on_remover" ]] || (( start - on_remover < 5 )); then @@ -3711,6 +3718,7 @@ test_snapshot_delete_after_overwrite() { ssh_run 1 "rm -f $file" ssh_run 1 "head -c 2097152 /dev/urandom > $file" || return 1 + sleep "$RESERVE_IDLE_RETURN_SECS" free_before=$(ssh_run 1 "stat -f -c %f /mnt/goclusterfs") ssh_run 1 "setfattr -n user.snapshot -v snap_delete_temp $file" || { log_error "snapshot failed" From 81899c23ca0cd283e00f08608663f5b074a3a4e0 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 03:05:52 -0600 Subject: [PATCH 4/7] fix(reserve): never hold reserve locks across the bitmap lock or I/O Review of #142 found that Take held the reserve's mutexes while waiting for the cluster bitmap lock, and writers took the same mutexes under an inode lock. A node dying with the bitmap lock could then hang the cluster: its replay needs an inode lock that a writer holds while it waits behind the refill. - The reserve mutex guards only memory. Refill, return, renew and record ends run outside it; writers never wait for them. - Each refill is its own record. A record ends once all its blocks are used, or early when the reserve goes back or falls far behind the journal tail. A crash then leaks in-flight blocks, never frees twice. - Blocks are freed only after their record end is synced (CompleteIntentDurable keeps the intent pending until then). - A failed bitmap write in a refill ends the record, then restores the sectors, under the bitmap lock. - A failed use commit leaves its blocks allocated instead of reusing them; failed commits are aborted. - Peer return requests set a flag the reserve's tick reads, so a flood costs one return per second. The asker re-asks every second and waits 12s, longer than an idle return. - Statfs clamps free blocks to the total. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- cmd/goclusterfs-mount/main.go | 2 +- docs/OPERATIONS.md | 6 +- internal/bitmap/bitmap.go | 33 +- internal/bitmap/bitmap_test.go | 40 ++ internal/cache/broadcast/receiver.go | 4 +- internal/goclusterfs/fs.go | 15 +- internal/goclusterfs/reserve_test.go | 56 ++- internal/journal/journal.go | 23 ++ internal/journal/reserve_test.go | 25 ++ internal/reserve/reserve.go | 522 +++++++++++++++++++-------- internal/reserve/reserve_test.go | 328 ++++++++++++++--- tests/test-manifest.yaml | 84 ++++- 13 files changed, 904 insertions(+), 236 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a875cc..87a0b4d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Read a snapshot by cloning it with `user.snapshot.clone`. - `goclusterfs-fsck` consistency checker. `-repair` rebuilds block refcounts from every file, snapshot and clone. It refuses to run while the device is in use, and `-repair -dry-run` shows what it would change. Exit codes follow fsck(8): `0` no errors, `1` repaired, `4` errors remain, `8` failed or refused, `16` usage error. - Benchmark suite (`tests/benchmark`) and an `internal/metrics` package. -- Per-node block reserves (`-block-reserve`, default 64): each node takes a batch of free blocks under one hold of the bitmap lock and hands them to its writes without it, gives unused ones back when idle, at unmount and when space runs low, and waits up to 10 seconds for other nodes' reserves before reporting `ENOSPC`. Journal replay and fsck return the unused reserve of a crashed node. See OPERATIONS.md. +- Per-node block reserves (`-block-reserve`, default 64): each node takes a batch of free blocks under one hold of the bitmap lock and hands them to its writes without it, gives unused ones back when idle, at unmount and when space runs low, and waits up to 12 seconds for other nodes' reserves before reporting `ENOSPC`. Journal replay and fsck return the unused reserve of a crashed node. See OPERATIONS.md. - `/metrics` and `/api/v1/stats/lock` report, per lock type, the acquires served, the acquires that found the lock taken, and how long those waited (`goclusterfs_lock_acquires_total`, `goclusterfs_lock_blocked_total`, `goclusterfs_lock_wait_seconds_total`). - Go, Docker, and QEMU VM test suites, with `tests/test-manifest.yaml` and `make test-parity` to track coverage across them. - Fuzz tests for the on-disk decoders (superblock, inode, snapshot descriptor, journal superblock and records) and the journal recovery scan. diff --git a/cmd/goclusterfs-mount/main.go b/cmd/goclusterfs-mount/main.go index 10f0ec37..9d11d8c7 100644 --- a/cmd/goclusterfs-mount/main.go +++ b/cmd/goclusterfs-mount/main.go @@ -443,7 +443,7 @@ func main() { return len(active) }) res.SetBroadcast(cacheMgr.BroadcastReturnReserves) - cacheMgr.OnReturnReserves(func() { res.ReturnAll(context.Background()) }) + cacheMgr.OnReturnReserves(res.AskReturn) bm.SetNoSpaceWait(res) j.SetCheckpointHook(func() { res.Refresh(context.Background()) }) gfs.SetReserve(res) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 57fde59a..2a76b3d0 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -311,10 +311,10 @@ What is safe today: a node that crashes or is killed and stays down, and a clean Each node takes up to 64 free blocks from the block bitmap at once (`-block-reserve`), under one hold of the cluster-wide bitmap lock, and hands them to its writes without that lock. Without reserves, every new data block takes the bitmap lock, and with four nodes writing at once about 60% of their time went to waiting for it. Only new data blocks of writes come from the reserve. Extent-tree blocks, directory blocks, copy-on-write copies and `fallocate` still take the bitmap lock. - **Size.** A reserve holds at most `-block-reserve` blocks, and at most 1/16 of the free blocks per mounted node. There are no reserves while fewer blocks are free than 1% of the device or two reserves per mounted node. -- **Going back.** A node gives its unused reserve back after 10 seconds without a write, at a clean unmount, and when a write finds free space below that line. -- **Full device.** A node that finds no free block gives back its own reserve, asks the other nodes over broadcast to give back theirs, and retries for up to 10 seconds before it fails with `ENOSPC`. It logs the wait at INFO, and `goclusterfs_reserve_peer_waits_total` and `goclusterfs_reserve_peer_wait_seconds_total` count it. Without broadcast, other nodes still give back idle reserves within 10 seconds. +- **Going back.** A node gives its unused reserve back within 11 seconds of its last write, at a clean unmount, and when a write finds free space below that line. +- **Full device.** A node that finds no free block gives back its own reserve, asks the other nodes over broadcast to give back theirs, once a second, and retries for up to 12 seconds before it fails with `ENOSPC`. It logs the wait at INFO, and `goclusterfs_reserve_peer_waits_total` and `goclusterfs_reserve_peer_wait_seconds_total` count it. Without broadcast, other nodes still give back idle reserves within 11 seconds. A node answers at most one request a second. - **`df`.** A node counts its own unused reserve as free, since its writes can use it. Other nodes' reserves count as used until they give them back, so `df` on two nodes can differ by up to 64 MB per node at the default. -- **Crash.** Each reserve is a journal record. When the node's journal is replayed, by the node at its next mount or by another node, the blocks it reserved and never used go back to the bitmap. fsck reports them as "reserved by node N and never used", which is not an error, and `fsck -repair -force` frees them after it marks the reserve done in that node's journal, so a later replay does not free them again. A crash in a few narrow windows, such as between ending a reserve and freeing it, leaks a few blocks instead; `fsck -repair` reclaims them. +- **Crash.** Each reserve is a journal record. When the node's journal is replayed, by the node at its next mount or by another node, the blocks it reserved and never used go back to the bitmap. fsck reports them as "reserved by node N and never used", which is not an error, and `fsck -repair -force` frees them after it marks the reserve done in that node's journal, so a later replay does not free them again. A crash in a few narrow windows leaks a few blocks instead, and `fsck -repair` reclaims them: between ending a reserve and freeing it, and while a write holds a block whose reserve the node ended early, to give it back or because the journal ring was filling up. ### Handling Node Failure diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index 7a4841f6..f6321a03 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -366,8 +366,11 @@ func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { } } -// Reserve calls record before marking the blocks allocated, and returns them even if marking fails. -func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) error) ([]uint64, error) { +// Reserve picks up to n free blocks under one lock hold and calls record with +// them. Only if record reports keep does it mark them allocated. If a sector +// write then fails, it calls end and, only if end succeeds, restores the sectors +// it changed, so no block a pending record lists is free on disk. +func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) (keep bool), end func() error) ([]uint64, error) { if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { return nil, fmt.Errorf("failed to acquire bitmap lock: %w", err) @@ -379,12 +382,13 @@ func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) error numSectors := (r.totalBlocks + BitsPerSector - 1) / BitsPerSector var picked, changed []uint64 - sectors := make(map[uint64][]byte) + sectors, originals := make(map[uint64][]byte), make(map[uint64][]byte) for sectorIdx := uint64(0); sectorIdx < numSectors && len(picked) < n; sectorIdx++ { data, err := r.readSector(sectorIdx) if err != nil { return nil, fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err) } + original := slices.Clone(data) before := len(picked) for bit := uint64(0); bit < BitsPerSector && len(picked) < n; bit++ { block := sectorIdx*BitsPerSector + bit @@ -397,20 +401,31 @@ func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) error } } if len(picked) > before { - sectors[sectorIdx] = data + sectors[sectorIdx], originals[sectorIdx] = data, original changed = append(changed, sectorIdx) } } if len(picked) == 0 { return nil, ErrNoFreeBlocks } - if err := record(picked); err != nil { - return nil, err + if !record(picked) { + return nil, errors.New("reserve not recorded") } - for _, sectorIdx := range changed { - if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil { - return picked, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + for i, sectorIdx := range changed { + err := r.writeSector(sectorIdx, sectors[sectorIdx]) + if err == nil { + continue + } + if eerr := end(); eerr != nil { + log.Printf("[ERROR] bitmap: reserved blocks %v may be free on disk while their record is pending: %v", picked, eerr) + return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + for _, done := range changed[:i+1] { + if rerr := r.writeSector(done, originals[done]); rerr != nil { + log.Printf("[WARN] bitmap: restoring sector %d after a failed reserve failed, its picked blocks stay allocated: %v", done, rerr) + } } + return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) } return picked, nil } diff --git a/internal/bitmap/bitmap_test.go b/internal/bitmap/bitmap_test.go index 6e32b15c..9897523c 100644 --- a/internal/bitmap/bitmap_test.go +++ b/internal/bitmap/bitmap_test.go @@ -729,3 +729,43 @@ func TestFreeIfAndHoldAllocated(t *testing.T) { } } } + +// A reserve whose second sector write fails ends its record and then restores +// the first sector, so no picked block stays allocated. If the record cannot be +// ended, the sectors stay as they are. +func TestReserveRestoresSectorsWhenAWriteFails(t *testing.T) { + for _, endErr := range []error{nil, errors.New("journal failed")} { + dev, cleanup := createTestDevice(t, 10<<20) + var uuid [16]byte + region := NewRegion(dev, 0, BitsPerSector+100, nil, uuid) + ctx := context.Background() + if _, err := region.Alloc(ctx, BitsPerSector-8); err != nil { + t.Fatal(err) + } + free := region.FreeCount() + failed := false + dev.SetWriteGate(func(offset int64) error { + if offset == SectorSize && !failed { + failed = true + return errors.New("injected write error") + } + return nil + }) + ended := false + blocks, err := region.Reserve(ctx, 16, func([]uint64) bool { return true }, func() error { + ended = true + return endErr + }) + if err == nil || !ended { + t.Fatalf("end error %v: Reserve returned %v, %v and ended=%v after the failed sector write", endErr, blocks, err, ended) + } + want := free + if endErr != nil { + want = free - 8 + } + if got := region.FreeCount(); got != want { + t.Errorf("end error %v: %d free blocks after the failed reserve, want %d", endErr, got, want) + } + cleanup() + } +} diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index 0c332740..a56ff7ca 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -191,12 +191,12 @@ func (b *Broadcaster) handleMessage(msg *Message, addr *net.UDPAddr) { case TypeReturnReserves: if fn := b.onReturnReserves.Load(); fn != nil { - go (*fn)() + (*fn)() } } } -// OnReturnReserves sets what runs when a peer asks for reserved blocks back. +// OnReturnReserves sets what runs, on the receive loop, when a peer asks for reserved blocks back; fn must not block. func (b *Broadcaster) OnReturnReserves(fn func()) { b.onReturnReserves.Store(&fn) } diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index d8465752..ca6f0a36 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -597,7 +597,7 @@ func (f *FS) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno { totalBlocks := f.bitmapMgr.TotalBlocks() freeBlocks := f.bitmapMgr.FreeCount() if f.reserve != nil { - freeBlocks += uint64(f.reserve.Stats().Blocks) + freeBlocks = min(totalBlocks, freeBlocks+uint64(f.reserve.Stats().Blocks)) } out.Blocks = totalBlocks @@ -2059,14 +2059,17 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i return err } var err error - seq, err = f.fs.journal.Commit(txn) + if seq, err = f.fs.journal.Commit(txn); err != nil { + f.fs.journal.Abort(txn) + } return err }) + used := len(reserveOut) + reserveOut = nil if err != nil { - logging.Debugf("Write inode %d: logging the use of %d reserved blocks failed: %v", f.inodeNum, len(reserveOut), err) + logging.Debugf("Write inode %d: logging the use of %d reserved blocks failed, leaving them allocated: %v", f.inodeNum, used, err) return syscall.EIO } - reserveOut = nil intents = append(intents, seq) return nil } @@ -2584,9 +2587,7 @@ func (f *File) writeInternal(ctx context.Context, data []byte, offset uint64) (i intentSeq, err = f.fs.journal.Commit(txn) return err }) - if commitErr == nil { - reserveOut = nil - } + reserveOut = nil } else { intentSeq, commitErr = f.fs.journal.Commit(txn) } diff --git a/internal/goclusterfs/reserve_test.go b/internal/goclusterfs/reserve_test.go index 306c046c..fa19aedc 100644 --- a/internal/goclusterfs/reserve_test.go +++ b/internal/goclusterfs/reserve_test.go @@ -12,6 +12,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/fsck" "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/reserve" "github.com/supporttools/GoClusterFS/internal/superblock" ) @@ -215,6 +216,59 @@ func TestReplayKeepsMappedBlocksOfCrashedNodesReserve(t *testing.T) { } } +// Node B holds the bitmap lock while node A's refill waits for it. Statfs and a +// write that maps a block under its inode lock on A must not wait for the refill: +// if B died holding the lock, its replay could need that inode lock. +func TestInodeLockHoldersDoNotWaitForRefill(t *testing.T) { + nodeA, path, _, _ := reserveFS(t) + nodeB := openFormattedFS(t, path) + resA := withReserve(t, nodeA, 8, time.Hour) + f := createFile(t, nodeA, "f") + writeAt(t, f, pattern(1, 4096), 0) + expectRefills(t, resA) + var held []uint64 + for { + b, ok := resA.TakeReady() + if !ok { + break + } + held = append(held, b) + } + + ctx := context.Background() + if err := nodeB.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { + t.Fatal(err) + } + refilled := make(chan struct{}) + go func() { + defer close(refilled) + resA.Take(ctx) + }() + time.Sleep(100 * time.Millisecond) + + done := make(chan struct{}) + go func() { + defer close(done) + var st fuse.StatfsOut + nodeA.Statfs(ctx, &st) + if err := nodeA.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { + t.Error(err) + return + } + resA.Untake(held[0]) + resA.TakeReady() + _ = resA.Logged(held[1:], func() error { return nil }) + nodeA.release(lock.LockTypeInode, uint64(f.inodeNum), "inode") + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("statfs or an inode-lock holder waited for the refill") + } + nodeB.release(lock.LockTypeBlockBitmap, 0, "bitmap") + <-refilled +} + // Node B finds the bitmap full while node A holds blocks in reserve. B asks for // them back and waits; A gives them back, and B's write gets a block instead of // failing with ENOSPC. @@ -227,7 +281,7 @@ func TestFullBitmapWaitsForPeersReserve(t *testing.T) { resA := withReserve(t, nodeA, 4, time.Hour) resB := withReserve(t, nodeB, 4, time.Hour) defer expectRefills(t, resA) - resB.SetBroadcast(func() { go resA.ReturnAll(context.Background()) }) + resB.SetBroadcast(resA.AskReturn) writeAt(t, createFile(t, nodeA, "a"), pattern(1, 4096), 0) if resA.Stats().Blocks == 0 { t.Fatal("node A holds no reserve") diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 03ae3a3d..facbd1e0 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -560,7 +560,30 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { j.mu.Lock() defer j.mu.Unlock() + return j.appendCompletionLocked(intentSeq) +} + +// CompleteIntentDurable returns only after the completion record is synced, +// and keeps the intent pending on error, so a retry writes the record. +func (j *Journal) CompleteIntentDurable(intentSeq uint64) error { + j.mu.Lock() + _, pending := j.pendingIntents[intentSeq] + var err error + if pending { + err = j.appendCompletionLocked(intentSeq) + } + j.mu.Unlock() + if !pending || err != nil { + return err + } + if err := j.region.Sync(); err != nil { + return fmt.Errorf("failed to sync intent completion: %w", err) + } + j.dropIntent(intentSeq) + return nil +} +func (j *Journal) appendCompletionLocked(intentSeq uint64) error { need := sectorAligned(completionLength) if need > j.region.AvailableSpace() { if err := j.forceCheckpoint(); err != nil || need > j.region.AvailableSpace() { diff --git a/internal/journal/reserve_test.go b/internal/journal/reserve_test.go index 14904659..ef0ef371 100644 --- a/internal/journal/reserve_test.go +++ b/internal/journal/reserve_test.go @@ -1,6 +1,7 @@ package journal import ( + "errors" "slices" "testing" ) @@ -180,3 +181,27 @@ func TestCompleteReservesStopsReplayFreeingThem(t *testing.T) { t.Errorf("replay would still free %v", got) } } + +// CompleteIntentDurable keeps the intent pending when the completion cannot be +// written, so the retry writes it and replay then skips the intent. +func TestCompleteIntentDurableRetriesAfterFailure(t *testing.T) { + j := newTestJournal(t) + seq := commitIntents(t, j, false)[0] + j.region.dev.SetWriteGate(func(int64) error { return errors.New("injected write error") }) + if err := j.CompleteIntentDurable(seq); err == nil { + t.Fatal("CompleteIntentDurable hid the write error") + } + if _, _, pending := j.IntentLag(seq); !pending { + t.Fatal("the intent stopped being pending although its completion was not written") + } + j.region.dev.SetWriteGate(func(int64) error { return nil }) + if err := j.CompleteIntentDurable(seq); err != nil { + t.Fatal(err) + } + if _, _, pending := j.IntentLag(seq); pending { + t.Error("the intent is still pending after its completion was written") + } + if got := replayedAfterCrash(t, j); len(got) != 0 { + t.Errorf("replayed %v after a crash, want nothing", got) + } +} diff --git a/internal/reserve/reserve.go b/internal/reserve/reserve.go index 2ad1460d..eeba1765 100644 --- a/internal/reserve/reserve.go +++ b/internal/reserve/reserve.go @@ -16,12 +16,13 @@ import ( const ( DefaultSize = 64 DefaultIdle = 10 * time.Second - // MaxPeerWait bounds how long a full bitmap waits for peers' reserves before ENOSPC. - MaxPeerWait = 10 * time.Second + tick = time.Second + // MaxPeerWait covers a peer's idle return, so a lost broadcast cannot cause ENOSPC. + MaxPeerWait = DefaultIdle + 2*tick ) type Bitmap interface { - Reserve(ctx context.Context, n int, record func([]uint64) error) ([]uint64, error) + Reserve(ctx context.Context, n int, record func([]uint64) (keep bool), end func() error) ([]uint64, error) FreeCount() uint64 TotalBlocks() uint64 } @@ -29,7 +30,8 @@ type Bitmap interface { type Journal interface { Begin() *journal.Transaction Commit(txn *journal.Transaction) (uint64, error) - CompleteIntent(seq uint64) error + Abort(txn *journal.Transaction) + CompleteIntentDurable(seq uint64) error IntentLag(seq uint64) (lag, size uint32, ok bool) } @@ -46,141 +48,219 @@ type Stats struct { WaitNanos uint64 // time those allocations waited } +type state int + +const ( + open state = iota + ending + ended +) + +// A record is one BlockReserve journal entry; its blocks are freed only after it ended. +type record struct { + seq uint64 + live int // its blocks that are untaken or handed out + state state + back []uint64 // handed back while ending +} + +// Writers hold an inode lock when they call TakeReady, Untake and Logged, so mu +// never spans a cluster lock or I/O, and writers never take endMu. type Manager struct { cfg Config bm Bitmap j Journal - free func(ctx context.Context, blocks []uint64) + release func(ctx context.Context, blocks []uint64) liveNodes func() int broadcast func() - // Shared while a write logs a use, exclusive while the record changes. - commits sync.RWMutex - mu sync.Mutex - untaken []uint64 - out map[uint64]bool // handed out, use not logged yet - seq uint64 // the reserve record, 0 when there is none - touched time.Time - - freeCount uint64 - countedAt time.Time - nodes int - nodesAt time.Time + endMu sync.Mutex + + mu sync.Mutex + untaken []uint64 + owner map[uint64]*record // untaken and handed-out blocks + out map[uint64]bool + used []*record // all blocks used, end not written yet + stale []*record // ending failed; nothing is freed until they end + orphans []uint64 // handed back after their record ended + refilling bool + touched time.Time + + freeCount atomic.Uint64 + countedAt atomic.Int64 + nodes atomic.Int64 + nodesAt atomic.Int64 + askedAt atomic.Int64 + returnAsked atomic.Bool refills, returns, waits, waitNanos atomic.Uint64 stop chan struct{} done sync.WaitGroup } -func New(cfg Config, bm Bitmap, j Journal, free func(context.Context, []uint64), liveNodes func() int) *Manager { +func New(cfg Config, bm Bitmap, j Journal, release func(context.Context, []uint64), liveNodes func() int) *Manager { if cfg.Idle <= 0 { cfg.Idle = DefaultIdle } - return &Manager{cfg: cfg, bm: bm, j: j, free: free, liveNodes: liveNodes, out: make(map[uint64]bool)} + m := &Manager{cfg: cfg, bm: bm, j: j, release: release, liveNodes: liveNodes, + owner: make(map[uint64]*record), out: make(map[uint64]bool)} + m.refreshCounts() + return m } func (m *Manager) SetBroadcast(fn func()) { m.broadcast = fn } -// Take reports false when reserves are off or space is low; the caller then uses the bitmap. +// Take reports false when reserves are off or space is low; the caller then uses +// the bitmap. Refilling takes the bitmap lock, so the caller must not hold an inode lock. func (m *Manager) Take(ctx context.Context) (uint64, bool) { if m.cfg.Size == 0 { return 0, false } - m.mu.Lock() - if len(m.untaken) > 0 && !m.lowSpaceLocked() { - defer m.mu.Unlock() - return m.handOutLocked(), true - } - m.mu.Unlock() - - m.commits.Lock() - defer m.commits.Unlock() - m.mu.Lock() - defer m.mu.Unlock() - if m.lowSpaceLocked() { - m.returnLocked(ctx) + m.refreshCounts() + if m.lowSpace() { + m.ReturnAll(ctx) return 0, false } - if len(m.untaken) == 0 && !m.refillLocked(ctx) { + if b, ok := m.TakeReady(); ok { + return b, true + } + if !m.refill(ctx) { return 0, false } - return m.handOutLocked(), true + return m.TakeReady() } -// TakeReady never refills, since refilling takes the bitmap lock and callers hold an inode lock. func (m *Manager) TakeReady() (uint64, bool) { - if m.cfg.Size == 0 { + if m.cfg.Size == 0 || m.lowSpace() { return 0, false } m.mu.Lock() defer m.mu.Unlock() - if len(m.untaken) == 0 || m.lowSpaceLocked() { + if len(m.untaken) == 0 { return 0, false } - return m.handOutLocked(), true -} - -func (m *Manager) handOutLocked() uint64 { m.touched = time.Now() b := m.untaken[0] m.untaken = m.untaken[1:] m.out[b] = true - return b + return b, true } +// Untake puts back blocks a write took and never tried to log. func (m *Manager) Untake(blocks ...uint64) { m.mu.Lock() defer m.mu.Unlock() for _, b := range blocks { - if m.out[b] { - delete(m.out, b) + if !m.out[b] { + continue + } + delete(m.out, b) + switch r := m.owner[b]; r.state { + case open: m.untaken = append(m.untaken, b) + case ending: + r.back = append(r.back, b) + case ended: + delete(m.owner, b) + r.live-- + m.orphans = append(m.orphans, b) } } slices.Sort(m.untaken) } -// Logged runs log, which commits the use of blocks, while the reserve record cannot change. +// Logged runs log, which commits the use of blocks. They leave the reserve even +// if log fails, since a failed commit may still be on disk. func (m *Manager) Logged(blocks []uint64, log func() error) error { - m.commits.RLock() - defer m.commits.RUnlock() - if err := log(); err != nil { - return err - } + err := log() m.mu.Lock() defer m.mu.Unlock() for _, b := range blocks { + if !m.out[b] { + continue + } delete(m.out, b) + r := m.owner[b] + delete(m.owner, b) + r.live-- + if r.live == 0 && r.state == open { + r.state = ending + m.used = append(m.used, r) + } } - return nil + return err } -// ReturnAll frees the untaken blocks; blocks still handed out move to a new record. +// ReturnAll gives the untaken blocks back. Each record ends before its blocks are +// freed, so a crash in between leaks them instead of replay freeing them twice. func (m *Manager) ReturnAll(ctx context.Context) { - m.commits.Lock() - defer m.commits.Unlock() - m.mu.Lock() - defer m.mu.Unlock() - m.returnLocked(ctx) + m.endMu.Lock() + defer m.endMu.Unlock() + ctx = context.WithoutCancel(ctx) + m.endUsed() + if !m.endStale() { + return + } + groups := m.takeUntaken(func(*record) bool { return true }, false) + for r, blocks := range groups { + if err := m.j.CompleteIntentDurable(r.seq); err != nil { + log.Printf("[reserve] keeping %d reserved blocks: ending record %d failed: %v", len(blocks), r.seq, err) + m.reopen(r, blocks) + continue + } + m.finishEnd(r) + if len(blocks) > 0 { + m.release(ctx, blocks) + m.freeCount.Add(uint64(len(blocks))) + } + } + if len(groups) > 0 { + m.returns.Add(1) + } + m.freeOrphans(ctx) } -// Refresh re-logs a record that has fallen far behind the tail, so it stops pinning the journal head. +// Refresh moves the untaken blocks of records far behind the journal tail into a +// new record and ends the old ones, so they stop holding the journal head back. +// A crash then leaks the old records' handed-out blocks instead of freeing them. func (m *Manager) Refresh(ctx context.Context) { - m.commits.Lock() - defer m.commits.Unlock() - m.mu.Lock() - defer m.mu.Unlock() - if m.seq == 0 { + if !m.endMu.TryLock() { return } - lag, size, ok := m.j.IntentLag(m.seq) - if !ok || lag < size/4 { + defer m.endMu.Unlock() + m.endUsed() + m.endStale() + old := m.oldRecords() + if len(old) == 0 { return } - if err := m.relogLocked(append(slices.Clone(m.untaken), m.outBlocks()...)); err != nil { - log.Printf("[reserve] renewing the reserve record failed: %v", err) + moved := m.takeUntaken(func(r *record) bool { return old[r] }, true) + var blocks []uint64 + for _, bs := range moved { + blocks = append(blocks, bs...) + } + if len(blocks) > 0 { + seq, err := m.commitRecord(blocks) + if err != nil { + log.Printf("[reserve] renewing %d reserved blocks failed: %v", len(blocks), err) + for r, bs := range moved { + m.reopen(r, bs) + } + return + } + m.add(seq, blocks) + } + for r := range moved { + if err := m.j.CompleteIntentDurable(r.seq); err != nil { + log.Printf("[reserve] ending renewed record %d failed: %v", r.seq, err) + m.mu.Lock() + m.stale = append(m.stale, r) + m.mu.Unlock() + continue + } + m.finishEnd(r) } } @@ -198,9 +278,10 @@ func (m *Manager) WaitForPeers(ctx context.Context, waited time.Duration) bool { if waited == 0 { log.Printf("[INFO] [reserve] the bitmap is full; asking other nodes to give back their reserved blocks") m.ReturnAll(ctx) - if m.broadcast != nil { - m.broadcast() - } + } + if now := time.Now(); m.broadcast != nil && now.Sub(time.Unix(0, m.askedAt.Load())) >= tick { + m.askedAt.Store(now.UnixNano()) + m.broadcast() } select { case <-ctx.Done(): @@ -216,24 +297,29 @@ func (m *Manager) Waited(d time.Duration) { log.Printf("[INFO] [reserve] got blocks after waiting %v for other nodes' reserves", d.Round(time.Millisecond)) } +// AskReturn gives the reserve back at the next tick, so a flood of peer requests costs one return per tick. +func (m *Manager) AskReturn() { + m.returnAsked.Store(true) +} + func (m *Manager) Start() { if m.cfg.Size == 0 { return } m.stop = make(chan struct{}) m.done.Go(func() { - tick := time.NewTicker(m.cfg.Idle / 2) - defer tick.Stop() + t := time.NewTicker(tick) + defer t.Stop() for { select { case <-m.stop: return - case <-tick.C: - m.mu.Lock() - idle := m.seq != 0 && len(m.out) == 0 && time.Since(m.touched) >= m.cfg.Idle - m.mu.Unlock() - if idle { + case <-t.C: + m.refreshCounts() + if m.returnAsked.Swap(false) || m.idle() { m.ReturnAll(context.Background()) + } else { + m.tidy(context.Background()) } } } @@ -256,122 +342,248 @@ func (m *Manager) Stats() Stats { return Stats{Blocks: n, Refills: m.refills.Load(), Returns: m.returns.Load(), Waits: m.waits.Load(), WaitNanos: m.waitNanos.Load()} } -func (m *Manager) lowSpaceLocked() bool { - free, nodes := m.freeCountLocked(), m.liveNodesLocked() - return free < max(m.bm.TotalBlocks()/100, uint64(2*m.cfg.Size*nodes)) -} +func (m *Manager) refill(ctx context.Context) bool { + m.mu.Lock() + if m.refilling || len(m.untaken) > 0 { + ready := len(m.untaken) > 0 + m.mu.Unlock() + return ready + } + m.refilling = true + m.mu.Unlock() + defer func() { + m.mu.Lock() + m.refilling = false + m.mu.Unlock() + }() -// Blocks still handed out move into the new record. -func (m *Manager) refillLocked(ctx context.Context) bool { - n := min(m.cfg.Size, int(m.freeCountLocked()/uint64(16*m.liveNodesLocked()))) + n := min(m.cfg.Size, int(m.freeCount.Load()/uint64(16*m.nodes.Load()))) if n <= 0 { return false } - carried := m.outBlocks() - var newSeq uint64 - blocks, err := m.bm.Reserve(ctx, n, func(picked []uint64) error { - var err error - newSeq, err = m.commitRecord(append(slices.Clone(picked), carried...)) - return err + var seq uint64 + var recordErr, endErr error + blocks, err := m.bm.Reserve(ctx, n, func(picked []uint64) bool { + seq, recordErr = m.commitRecord(picked) + return !errors.Is(recordErr, journal.ErrJournalFull) + }, func() error { + if endErr = recordErr; endErr == nil { + endErr = m.j.CompleteIntentDurable(seq) + } + return endErr }) - if err != nil { - if newSeq != 0 { - m.endLocked(newSeq, blocks) + switch { + case recordErr != nil: + if !errors.Is(recordErr, journal.ErrJournalFull) { + log.Printf("[reserve] leaving %d blocks allocated, since their record may be on disk: %v", len(blocks), recordErr) } + return false + case err != nil: if !errors.Is(err, context.Canceled) { log.Printf("[reserve] taking %d blocks failed: %v", n, err) } - return false - } - if m.seq != 0 { - if err := m.j.CompleteIntent(m.seq); err != nil { - log.Printf("[reserve] ending reserve record %d failed: %v", m.seq, err) + if endErr != nil { + m.mu.Lock() + m.stale = append(m.stale, &record{seq: seq, state: ending}) + m.mu.Unlock() } + return false } - m.seq, m.untaken = newSeq, blocks - m.freeCount -= uint64(len(blocks)) + m.add(seq, blocks) + m.freeCount.Add(^uint64(len(blocks) - 1)) m.refills.Add(1) return true } -// Ending the record before freeing means a crash leaks blocks instead of replay double-freeing them. -func (m *Manager) returnLocked(ctx context.Context) { - if m.seq == 0 { - return +func (m *Manager) add(seq uint64, blocks []uint64) { + r := &record{seq: seq, live: len(blocks)} + m.mu.Lock() + defer m.mu.Unlock() + for _, b := range blocks { + m.owner[b] = r } - untaken := m.untaken - m.untaken = nil - if out := m.outBlocks(); len(out) > 0 { - if err := m.relogLocked(out); err != nil { - log.Printf("[reserve] keeping %d reserved blocks: renewing the record failed: %v", len(untaken), err) - m.untaken = untaken - return + m.untaken = append(m.untaken, blocks...) + slices.Sort(m.untaken) +} + +// takeUntaken removes the untaken blocks of the picked records and marks those +// records ending; withOut also picks records whose blocks are all handed out. +func (m *Manager) takeUntaken(pick func(*record) bool, withOut bool) map[*record][]uint64 { + m.mu.Lock() + defer m.mu.Unlock() + candidates := make(map[*record]bool) + if withOut { + for _, r := range m.owner { + candidates[r] = true } } else { - if err := m.j.CompleteIntent(m.seq); err != nil { - log.Printf("[reserve] keeping %d reserved blocks: ending the record failed: %v", len(untaken), err) - m.untaken = untaken - return + for _, b := range m.untaken { + candidates[m.owner[b]] = true + } + } + groups := make(map[*record][]uint64) + for r := range candidates { + if r.state == open && pick(r) { + r.state = ending + groups[r] = nil } - m.seq = 0 } - if len(untaken) > 0 { - m.free(ctx, untaken) - m.freeCount += uint64(len(untaken)) + kept := m.untaken[:0] + for _, b := range m.untaken { + r := m.owner[b] + if _, ok := groups[r]; !ok { + kept = append(kept, b) + continue + } + groups[r] = append(groups[r], b) + delete(m.owner, b) + r.live-- } - m.returns.Add(1) + m.untaken = kept + return groups } -func (m *Manager) relogLocked(blocks []uint64) error { - seq, err := m.commitRecord(blocks) - if err != nil { - return err +func (m *Manager) reopen(r *record, blocks []uint64) { + m.mu.Lock() + defer m.mu.Unlock() + r.state = open + r.live += len(blocks) + for _, b := range blocks { + m.owner[b] = r } - if err := m.j.CompleteIntent(m.seq); err != nil { - log.Printf("[reserve] ending reserve record %d failed: %v", m.seq, err) + m.untaken = append(m.untaken, blocks...) + m.untaken = append(m.untaken, r.back...) + r.back = nil + slices.Sort(m.untaken) +} + +func (m *Manager) finishEnd(r *record) { + m.mu.Lock() + defer m.mu.Unlock() + r.state = ended + for _, b := range r.back { + delete(m.owner, b) + r.live-- } - m.seq = seq - return nil + m.orphans = append(m.orphans, r.back...) + r.back = nil } -func (m *Manager) endLocked(seq uint64, blocks []uint64) { - if err := m.j.CompleteIntent(seq); err != nil { - log.Printf("[reserve] leaving %d blocks allocated: ending record %d failed: %v", len(blocks), seq, err) - return +func (m *Manager) endUsed() { + m.mu.Lock() + used := m.used + m.used = nil + m.mu.Unlock() + for _, r := range used { + if err := m.j.CompleteIntentDurable(r.seq); err != nil { + log.Printf("[reserve] ending used record %d failed: %v", r.seq, err) + m.mu.Lock() + m.stale = append(m.stale, r) + m.mu.Unlock() + continue + } + m.finishEnd(r) } - m.free(context.Background(), blocks) } -func (m *Manager) commitRecord(blocks []uint64) (uint64, error) { - txn := m.j.Begin() - if err := txn.LogBlockReserve(blocks); err != nil { - return 0, err +// endStale retries ending records whose end failed and reports whether none is left. +func (m *Manager) endStale() bool { + m.mu.Lock() + stale := m.stale + m.stale = nil + m.mu.Unlock() + for _, r := range stale { + if err := m.j.CompleteIntentDurable(r.seq); err != nil { + m.mu.Lock() + m.stale = append(m.stale, r) + m.mu.Unlock() + continue + } + m.finishEnd(r) } - return m.j.Commit(txn) + m.mu.Lock() + defer m.mu.Unlock() + if len(m.stale) > 0 { + log.Printf("[reserve] freeing no reserved blocks until %d unfinished records end", len(m.stale)) + } + return len(m.stale) == 0 +} + +func (m *Manager) freeOrphans(ctx context.Context) { + m.mu.Lock() + if len(m.stale) > 0 || len(m.orphans) == 0 { + m.mu.Unlock() + return + } + orphans := m.orphans + m.orphans = nil + m.mu.Unlock() + m.release(ctx, orphans) + m.freeCount.Add(uint64(len(orphans))) +} + +func (m *Manager) tidy(ctx context.Context) { + m.endMu.Lock() + defer m.endMu.Unlock() + m.endUsed() + m.endStale() + m.freeOrphans(ctx) +} + +func (m *Manager) idle() bool { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.untaken) > 0 && len(m.out) == 0 && time.Since(m.touched) >= m.cfg.Idle } -func (m *Manager) outBlocks() []uint64 { - out := make([]uint64, 0, len(m.out)) - for b := range m.out { - out = append(out, b) +func (m *Manager) oldRecords() map[*record]bool { + m.mu.Lock() + live := make(map[*record]bool) + for _, r := range m.owner { + if r.state == open { + live[r] = true + } } - slices.Sort(out) - return out + m.mu.Unlock() + old := make(map[*record]bool) + for r := range live { + if lag, size, ok := m.j.IntentLag(r.seq); ok && lag >= size/4 { + old[r] = true + } + } + return old +} + +func (m *Manager) lowSpace() bool { + nodes := uint64(m.nodes.Load()) + return m.freeCount.Load() < max(m.bm.TotalBlocks()/100, 2*uint64(m.cfg.Size)*nodes) } -func (m *Manager) freeCountLocked() uint64 { - if time.Since(m.countedAt) > 2*time.Second { - m.freeCount, m.countedAt = m.bm.FreeCount(), time.Now() +func (m *Manager) refreshCounts() { + now := time.Now() + if now.Sub(time.Unix(0, m.countedAt.Load())) > 2*time.Second { + m.freeCount.Store(m.bm.FreeCount()) + m.countedAt.Store(now.UnixNano()) + } + if now.Sub(time.Unix(0, m.nodesAt.Load())) > 10*time.Second { + n := 1 + if m.liveNodes != nil { + n = max(1, m.liveNodes()) + } + m.nodes.Store(int64(n)) + m.nodesAt.Store(now.UnixNano()) } - return m.freeCount } -func (m *Manager) liveNodesLocked() int { - if m.liveNodes == nil { - return 1 +func (m *Manager) commitRecord(blocks []uint64) (uint64, error) { + txn := m.j.Begin() + if err := txn.LogBlockReserve(blocks); err != nil { + m.j.Abort(txn) + return 0, err } - if time.Since(m.nodesAt) > 10*time.Second { - m.nodes, m.nodesAt = max(1, m.liveNodes()), time.Now() + seq, err := m.j.Commit(txn) + if err != nil { + m.j.Abort(txn) } - return m.nodes + return seq, err } diff --git a/internal/reserve/reserve_test.go b/internal/reserve/reserve_test.go index 81eb5306..97c42f69 100644 --- a/internal/reserve/reserve_test.go +++ b/internal/reserve/reserve_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "slices" + "sync" "testing" "time" @@ -11,8 +12,10 @@ import ( ) type fakeBitmap struct { + mu sync.Mutex total uint64 free []uint64 + gate chan struct{} // when set, Reserve waits on it, as for a busy bitmap lock } func newFakeBitmap(total, used uint64) *fakeBitmap { @@ -23,61 +26,138 @@ func newFakeBitmap(total, used uint64) *fakeBitmap { return b } -func (b *fakeBitmap) Reserve(_ context.Context, n int, record func([]uint64) error) ([]uint64, error) { +func (b *fakeBitmap) Reserve(_ context.Context, n int, record func([]uint64) bool, _ func() error) ([]uint64, error) { + if b.gate != nil { + <-b.gate + } + b.mu.Lock() + defer b.mu.Unlock() n = min(n, len(b.free)) picked := slices.Clone(b.free[:n]) - if err := record(picked); err != nil { - return nil, err + if !record(picked) { + return nil, errors.New("not recorded") } b.free = b.free[n:] return picked, nil } -func (b *fakeBitmap) FreeCount() uint64 { return uint64(len(b.free)) } +func (b *fakeBitmap) FreeCount() uint64 { + b.mu.Lock() + defer b.mu.Unlock() + return uint64(len(b.free)) +} + func (b *fakeBitmap) TotalBlocks() uint64 { return b.total } -func (b *fakeBitmap) give(_ context.Context, blocks []uint64) { + +func (b *fakeBitmap) give(blocks []uint64) { + b.mu.Lock() + defer b.mu.Unlock() b.free = append(b.free, blocks...) slices.Sort(b.free) } -// fakeJournal records the order of commits and completions. +// fakeJournal records the order of commits, completions and frees. type fakeJournal struct { - seq uint64 - events []string - completed map[uint64]bool - lag uint32 - failNext bool + mu sync.Mutex + seq uint64 + events []string + pending map[uint64]bool + lag uint32 + commitErr error + completeErr error + aborted int + freeingBlock chan struct{} // when set, frees wait on it, as for a busy bitmap lock +} + +func (j *fakeJournal) event(e string) { + j.mu.Lock() + defer j.mu.Unlock() + j.events = append(j.events, e) +} + +func (j *fakeJournal) take() []string { + j.mu.Lock() + defer j.mu.Unlock() + e := j.events + j.events = nil + return e } func (j *fakeJournal) Begin() *journal.Transaction { return journal.NewTransaction(0, [16]byte{1}, 1) } + func (j *fakeJournal) Commit(*journal.Transaction) (uint64, error) { - if j.failNext { - j.failNext = false - return 0, errors.New("commit failed") + j.mu.Lock() + defer j.mu.Unlock() + if j.commitErr != nil { + return 0, j.commitErr } j.seq++ + if j.pending == nil { + j.pending = map[uint64]bool{} + } + j.pending[j.seq] = true j.events = append(j.events, "commit") return j.seq, nil } -func (j *fakeJournal) CompleteIntent(seq uint64) error { - if j.completed == nil { - j.completed = map[uint64]bool{} + +func (j *fakeJournal) Abort(*journal.Transaction) { + j.mu.Lock() + defer j.mu.Unlock() + j.aborted++ +} + +func (j *fakeJournal) CompleteIntentDurable(seq uint64) error { + j.mu.Lock() + defer j.mu.Unlock() + if !j.pending[seq] { + return nil + } + if j.completeErr != nil { + return j.completeErr } - j.completed[seq] = true + delete(j.pending, seq) j.events = append(j.events, "complete") return nil } + func (j *fakeJournal) IntentLag(seq uint64) (uint32, uint32, bool) { - return j.lag, 1000, !j.completed[seq] && seq != 0 + j.mu.Lock() + defer j.mu.Unlock() + return j.lag, 1000, j.pending[seq] +} + +func (j *fakeJournal) set(f func(*fakeJournal)) { + j.mu.Lock() + defer j.mu.Unlock() + f(j) } func newManager(size int, bm *fakeBitmap, j *fakeJournal) *Manager { - return New(Config{Size: size, Idle: time.Hour}, bm, j, func(ctx context.Context, blocks []uint64) { - j.events = append(j.events, "free") - bm.give(ctx, blocks) + return New(Config{Size: size, Idle: time.Hour}, bm, j, func(_ context.Context, blocks []uint64) { + if j.freeingBlock != nil { + <-j.freeingBlock + } + j.event("free") + bm.give(blocks) }, func() int { return 1 }) } +func take(t *testing.T, m *Manager) uint64 { + t.Helper() + b, ok := m.Take(context.Background()) + if !ok { + t.Fatal("Take found no reserved block") + } + return b +} + +func expectEvents(t *testing.T, j *fakeJournal, what string, want ...string) { + t.Helper() + if got := j.take(); !slices.Equal(got, want) { + t.Errorf("%s did %v, want %v", what, got, want) + } +} + // A reserve holds no more than the configured size, nor a sixteenth of the free // blocks per node, and none at all below two reserves per node or 1% of the blocks. func TestReserveSize(t *testing.T) { @@ -113,66 +193,214 @@ func TestReserveSize(t *testing.T) { func TestReturnEndsRecordBeforeFreeing(t *testing.T) { bm, j := newFakeBitmap(10000, 0), &fakeJournal{} m := newManager(8, bm, j) - b, _ := m.Take(context.Background()) + b := take(t, m) if err := m.Logged([]uint64{b}, func() error { return nil }); err != nil { t.Fatal(err) } - j.events = nil + j.take() m.ReturnAll(context.Background()) - if want := []string{"complete", "free"}; !slices.Equal(j.events, want) { - t.Errorf("return did %v, want %v", j.events, want) - } + expectEvents(t, j, "return", "complete", "free") if got := bm.FreeCount(); got != 10000-1 { t.Errorf("%d free blocks after the return, want %d", got, 10000-1) } } -// A block a write took and has not logged keeps a record: returning the -// reserve writes a new record for it before ending the old one and freeing the rest. -func TestReturnKeepsHandedOutBlocksRecorded(t *testing.T) { +// Returning the reserve while a write holds one of its blocks ends the record +// anyway. If the write then hands the block back, it is freed without a record. +func TestBlockHandedBackAfterReturnIsFreed(t *testing.T) { bm, j := newFakeBitmap(10000, 0), &fakeJournal{} m := newManager(8, bm, j) - b, _ := m.Take(context.Background()) - j.events = nil + b := take(t, m) + j.take() m.ReturnAll(context.Background()) - if want := []string{"commit", "complete", "free"}; !slices.Equal(j.events, want) { - t.Errorf("return did %v, want %v", j.events, want) - } + expectEvents(t, j, "return", "complete", "free") m.Untake(b) - j.events = nil m.ReturnAll(context.Background()) - if want := []string{"complete", "free"}; !slices.Equal(j.events, want) || bm.FreeCount() != 10000 { - t.Errorf("second return did %v with %d free, want %v with all free", j.events, bm.FreeCount(), []string{"complete", "free"}) + expectEvents(t, j, "second return", "free") + if bm.FreeCount() != 10000 { + t.Errorf("%d free blocks, want all 10000", bm.FreeCount()) } } -// A write whose commit fails keeps its blocks in the reserve to put back. -func TestFailedLogKeepsBlocksOut(t *testing.T) { +// A block whose use commit failed may be mapped on disk, so it never goes back +// to the reserve or the bitmap. +func TestFailedLogLeavesBlockAllocated(t *testing.T) { bm, j := newFakeBitmap(10000, 0), &fakeJournal{} m := newManager(8, bm, j) - b, _ := m.Take(context.Background()) + b := take(t, m) if err := m.Logged([]uint64{b}, func() error { return errors.New("commit failed") }); err == nil { t.Fatal("Logged hid the commit error") } m.Untake(b) + if got := m.Stats().Blocks; got != 7 { + t.Errorf("%d blocks in reserve, want 7", got) + } + m.ReturnAll(context.Background()) + if got := bm.FreeCount(); got != 10000-1 { + t.Errorf("%d free blocks after the return, want %d", got, 10000-1) + } +} + +// When ending a record fails, its blocks stay reserved, and a later return ends it and frees them. +func TestFailedEndKeepsBlocks(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + m.Untake(take(t, m)) + j.take() + j.set(func(j *fakeJournal) { j.completeErr = journal.ErrJournalFull }) + m.ReturnAll(context.Background()) + expectEvents(t, j, "failed return") if got := m.Stats().Blocks; got != 8 { - t.Errorf("%d blocks in reserve after putting the block back, want 8", got) + t.Errorf("%d blocks in reserve after a failed return, want 8", got) } + j.set(func(j *fakeJournal) { j.completeErr = nil }) + m.ReturnAll(context.Background()) + expectEvents(t, j, "retried return", "complete", "free") } -// The record is written again once it is a quarter of the journal behind the tail. +// A record is renewed once it is a quarter of the journal behind the tail. If the +// old record cannot be ended, nothing is freed until it is, since it still lists the moved blocks. func TestRefreshRenewsAnOldRecord(t *testing.T) { bm, j := newFakeBitmap(10000, 0), &fakeJournal{} m := newManager(8, bm, j) - m.Take(context.Background()) - j.events, j.lag = nil, 100 + m.Untake(take(t, m)) + j.take() + j.set(func(j *fakeJournal) { j.lag = 100 }) m.Refresh(context.Background()) - if len(j.events) != 0 { - t.Errorf("refresh of a recent record did %v, want nothing", j.events) + expectEvents(t, j, "refresh of a recent record") + + j.set(func(j *fakeJournal) { j.lag, j.completeErr = 300, errors.New("sync failed") }) + m.Refresh(context.Background()) + expectEvents(t, j, "refresh with a failing end", "commit") + j.set(func(j *fakeJournal) { j.lag = 0 }) + m.ReturnAll(context.Background()) + expectEvents(t, j, "return while the old record is unfinished") + if got := m.Stats().Blocks; got != 8 { + t.Errorf("%d blocks in reserve, want 8", got) } - j.lag = 300 + + j.set(func(j *fakeJournal) { j.completeErr = nil }) + m.ReturnAll(context.Background()) + expectEvents(t, j, "return after the end works", "complete", "complete", "free") +} + +// A record whose blocks are all handed out and far behind the tail ends, so a +// write that waits long on a lock cannot hold the journal head back. +func TestRefreshEndsRecordOfHandedOutBlocks(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(2, bm, j) + take(t, m) + take(t, m) + j.take() + j.set(func(j *fakeJournal) { j.lag = 300 }) m.Refresh(context.Background()) - if want := []string{"commit", "complete"}; !slices.Equal(j.events, want) { - t.Errorf("refresh of an old record did %v, want %v", j.events, want) + expectEvents(t, j, "refresh", "complete") +} + +// A record whose blocks were all used ends at the next tidy. +func TestUsedRecordEnds(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(2, bm, j) + blocks := []uint64{take(t, m), take(t, m)} + if err := m.Logged(blocks, func() error { return nil }); err != nil { + t.Fatal(err) + } + j.take() + m.tidy(context.Background()) + expectEvents(t, j, "tidy", "complete") +} + +// A refill whose record hit a full journal marks nothing, since nothing was +// written. Any other commit error leaves the blocks allocated, since the record may be on disk. +func TestRefillRecordFailure(t *testing.T) { + for _, tc := range []struct { + err error + free uint64 + }{ + {journal.ErrJournalFull, 10000}, + {errors.New("sync failed"), 10000 - 8}, + } { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{commitErr: tc.err} + m := newManager(8, bm, j) + if _, ok := m.Take(context.Background()); ok { + t.Fatalf("%v: Take handed out a block without a record", tc.err) + } + if got := bm.FreeCount(); got != tc.free { + t.Errorf("%v: %d free blocks, want %d", tc.err, got, tc.free) + } + if j.aborted != 1 { + t.Errorf("%v: %d transactions aborted, want 1", tc.err, j.aborted) + } + } +} + +// Writes call TakeReady, Untake and Logged with an inode lock held, and statfs +// calls Stats. None of them may wait while a refill or a return waits for the +// bitmap lock, or a node that dies holding that lock hangs the cluster. +func TestWritersDoNotWaitForBitmapLock(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(2, bm, j) + first, second := take(t, m), take(t, m) + + writersFinish := func(what string) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + m.Stats() + m.TakeReady() + m.Untake(first) + if b, ok := m.TakeReady(); ok { + _ = m.Logged([]uint64{b}, func() error { return nil }) + } + _ = m.Logged([]uint64{second}, func() error { return nil }) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("writers waited while %s", what) + } + } + + bm.gate = make(chan struct{}) + refilled := make(chan struct{}) + go func() { + defer close(refilled) + m.Take(context.Background()) + }() + time.Sleep(50 * time.Millisecond) + writersFinish("a refill waited for the bitmap lock") + close(bm.gate) + <-refilled + bm.gate = nil + + j.freeingBlock = make(chan struct{}) + returned := make(chan struct{}) + go func() { + defer close(returned) + m.ReturnAll(context.Background()) + }() + time.Sleep(50 * time.Millisecond) + first, second = take(t, m), take(t, m) + writersFinish("a return waited for the bitmap lock") + close(j.freeingBlock) + <-returned +} + +// A flood of peer requests gives the reserve back at most once per tick, even +// while a writer keeps refilling it. +func TestPeerRequestsCoalesce(t *testing.T) { + bm, j := newFakeBitmap(100000, 0), &fakeJournal{} + m := newManager(8, bm, j) + m.Start() + defer m.Stop(context.Background()) + deadline := time.Now().Add(2*tick + tick/2) + for time.Now().Before(deadline) { + m.Untake(take(t, m)) + m.AskReturn() + time.Sleep(time.Millisecond) + } + if got := m.Stats().Returns; got < 1 || got > 3 { + t.Errorf("%d returns in 2.5 ticks of requests, want 1 to 3", got) } } diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 65b44954..8bf5485b 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3121,26 +3121,96 @@ scenarios: go: internal/reserve/reserve_test.go::TestReturnEndsRecordBeforeFreeing - id: reserve-return-handed-out - name: Returned reserve keeps handed-out blocks recorded - description: Blocks a write took and has not logged stay in a new record when the reserve is given back, and go back later + name: Block handed back after a return is freed + description: Returning the reserve while a write holds one of its blocks ends the record; the block the write later hands back is freed without a record priority: go_only environments: - go: internal/reserve/reserve_test.go::TestReturnKeepsHandedOutBlocksRecorded + go: internal/reserve/reserve_test.go::TestBlockHandedBackAfterReturnIsFreed - id: reserve-failed-log - name: Failed commit keeps reserved blocks - description: A write whose commit fails keeps its reserved blocks to put back + name: Failed commit leaves its reserved blocks allocated + description: A block whose use commit failed may be mapped on disk, so it goes neither back to the reserve nor to the bitmap priority: go_only environments: - go: internal/reserve/reserve_test.go::TestFailedLogKeepsBlocksOut + go: internal/reserve/reserve_test.go::TestFailedLogLeavesBlockAllocated + + - id: reserve-failed-end + name: Failed record end keeps its blocks + description: When ending a reserve record fails, its blocks stay reserved, and a later return ends it and frees them + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestFailedEndKeepsBlocks - id: reserve-refresh name: Old reserve record is renewed - description: The reserve record is written again once it is a quarter of the journal behind the tail + description: A record a quarter of the journal behind the tail moves its untaken blocks to a new record and ends; if the old record cannot end, nothing is freed until it does priority: go_only environments: go: internal/reserve/reserve_test.go::TestRefreshRenewsAnOldRecord + - id: reserve-refresh-handed-out + name: Old record of handed-out blocks ends + description: A record whose blocks are all handed out ends once it is far behind the tail, so a write stuck on a lock cannot hold the journal head back + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestRefreshEndsRecordOfHandedOutBlocks + + - id: reserve-used-record-ends + name: Fully used reserve record ends + description: A record whose blocks were all used ends at the next tidy + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestUsedRecordEnds + + - id: reserve-refill-record-failure + name: Refill whose record fails + description: A refill whose record hit a full journal marks nothing; any other commit error leaves the blocks allocated, and the transaction is aborted either way + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestRefillRecordFailure + + - id: reserve-writers-never-wait-for-bitmap + name: Writers never wait for the bitmap lock through the reserve + description: TakeReady, Untake, Logged and Stats return while a refill or a return waits for the bitmap lock + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestWritersDoNotWaitForBitmapLock + + - id: reserve-inode-lock-holders-skip-refill + name: Inode-lock holders and statfs do not wait for a refill + description: While another node holds the bitmap lock and a refill waits for it, statfs and a write holding an inode lock still use the reserve + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestInodeLockHoldersDoNotWaitForRefill + + - id: reserve-peer-requests-coalesce + name: Peer return requests coalesce + description: A flood of peer requests gives the reserve back at most once per tick while a writer keeps refilling it + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestPeerRequestsCoalesce + + - id: reserve-statfs-own-reserve + name: Statfs counts the node's own reserve as free + description: A one-block write lowers statfs by one block although it took a whole reserve from the bitmap + priority: go_only + environments: + go: internal/goclusterfs/reserve_test.go::TestStatfsCountsOwnReserveAsFree + + - id: bitmap-reserve-restores-on-write-failure + name: Failed reserve sector write ends the record, then restores + description: A reserve whose second sector write fails ends its record and restores the first sector; if the record cannot end, the sectors stay as they are + priority: go_only + environments: + go: internal/bitmap/bitmap_test.go::TestReserveRestoresSectorsWhenAWriteFails + + - id: journal-durable-intent-completion + name: Durable intent completion survives a failed write + description: CompleteIntentDurable keeps the intent pending when the completion cannot be written, so a retry writes it and replay skips the intent + priority: go_only + environments: + go: internal/journal/reserve_test.go::TestCompleteIntentDurableRetriesAfterFailure + - id: reserve-writes name: Writes take blocks from the reserve description: Ten one-block writes take the bitmap lock once, and unmount gives the rest of the reserve back; fsck is clean From fd150b948299454a03f3c9a6135e9197e6899b3b Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 03:25:17 -0600 Subject: [PATCH 5/7] test(reserve): check the statfs result in the refill test Co-Authored-By: Claude Opus 5.5 --- internal/goclusterfs/reserve_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/goclusterfs/reserve_test.go b/internal/goclusterfs/reserve_test.go index fa19aedc..a739713d 100644 --- a/internal/goclusterfs/reserve_test.go +++ b/internal/goclusterfs/reserve_test.go @@ -250,7 +250,9 @@ func TestInodeLockHoldersDoNotWaitForRefill(t *testing.T) { go func() { defer close(done) var st fuse.StatfsOut - nodeA.Statfs(ctx, &st) + if errno := nodeA.Statfs(ctx, &st); errno != 0 { + t.Error(errno) + } if err := nodeA.lockMgr.Acquire(ctx, lock.LockTypeInode, uint64(f.inodeNum), lock.LockModeExclusive); err != nil { t.Error(err) return From fa94361098a6eacec62d2ef7a051eb373dc2df7d Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 04:13:48 -0600 Subject: [PATCH 6/7] fix(reserve): end records in a full ring and keep frees out of endMu Second review of #142: - CompleteIntentDurable could never end a reserve record that is the oldest intent in a ring too full for a completion record, so every commit on the node then failed. It now moves the head past such an intent instead; a synced head past a record ends it for replay. - A record that a failed refresh reopened after its last blocks were used never ended. Reopen now queues it as fully used. - WaitForPeers called ReturnAll, which could wait on endMu while its holder waited for a bitmap claim that only the waiting replay frees. WaitForPeers now only asks for a return, and returns and tidies free blocks after dropping endMu, so its holders wait only for the journal. - When a refill's record cannot be ended after a failed bitmap write, Reserve writes the remaining sectors again and keeps the blocks. - A use commit that hit a full journal puts its blocks back. - The cached free count saturates at zero instead of wrapping. Co-Authored-By: Claude Opus 5.5 --- internal/bitmap/bitmap.go | 20 ++++-- internal/bitmap/bitmap_test.go | 13 ++-- internal/journal/journal.go | 35 +++++++++-- internal/journal/reserve_test.go | 41 ++++++++++++ internal/reserve/reserve.go | 71 ++++++++++++++------- internal/reserve/reserve_test.go | 104 +++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 37 ++++++++++- 7 files changed, 285 insertions(+), 36 deletions(-) diff --git a/internal/bitmap/bitmap.go b/internal/bitmap/bitmap.go index f6321a03..d67f17f7 100644 --- a/internal/bitmap/bitmap.go +++ b/internal/bitmap/bitmap.go @@ -368,8 +368,8 @@ func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) { // Reserve picks up to n free blocks under one lock hold and calls record with // them. Only if record reports keep does it mark them allocated. If a sector -// write then fails, it calls end and, only if end succeeds, restores the sectors -// it changed, so no block a pending record lists is free on disk. +// write then fails, it calls end and restores the sectors it changed; if end +// fails, it writes the rest again, so no block a pending record lists stays free on disk. func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) (keep bool), end func() error) ([]uint64, error) { if r.lockMgr != nil { if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil { @@ -417,8 +417,11 @@ func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) (keep continue } if eerr := end(); eerr != nil { - log.Printf("[ERROR] bitmap: reserved blocks %v may be free on disk while their record is pending: %v", picked, eerr) - return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + if rerr := r.writeSectors(changed[i:], sectors); rerr != nil { + log.Printf("[ERROR] bitmap: reserved blocks %v may be free on disk while their record is pending: %v, %v", picked, eerr, rerr) + return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + return picked, nil } for _, done := range changed[:i+1] { if rerr := r.writeSector(done, originals[done]); rerr != nil { @@ -430,6 +433,15 @@ func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) (keep return picked, nil } +func (r *Region) writeSectors(idx []uint64, sectors map[uint64][]byte) error { + for _, sectorIdx := range idx { + if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil { + return fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err) + } + } + return nil +} + // HoldAllocated takes the bitmap lock and returns the blocks of blocks that are // allocated. The lock stays held, so none of them can be freed, until release. // A block past the end of the volume is an error. diff --git a/internal/bitmap/bitmap_test.go b/internal/bitmap/bitmap_test.go index 9897523c..c2fbd0df 100644 --- a/internal/bitmap/bitmap_test.go +++ b/internal/bitmap/bitmap_test.go @@ -732,7 +732,7 @@ func TestFreeIfAndHoldAllocated(t *testing.T) { // A reserve whose second sector write fails ends its record and then restores // the first sector, so no picked block stays allocated. If the record cannot be -// ended, the sectors stay as they are. +// ended, it writes the failed sector again and keeps the blocks. func TestReserveRestoresSectorsWhenAWriteFails(t *testing.T) { for _, endErr := range []error{nil, errors.New("journal failed")} { dev, cleanup := createTestDevice(t, 10<<20) @@ -756,12 +756,15 @@ func TestReserveRestoresSectorsWhenAWriteFails(t *testing.T) { ended = true return endErr }) - if err == nil || !ended { - t.Fatalf("end error %v: Reserve returned %v, %v and ended=%v after the failed sector write", endErr, blocks, err, ended) + if !ended { + t.Fatalf("end error %v: Reserve did not end the record after the failed sector write", endErr) } - want := free + want, wantBlocks := free, 0 if endErr != nil { - want = free - 8 + want, wantBlocks = free-16, 16 + } + if (err == nil) != (endErr != nil) || len(blocks) != wantBlocks { + t.Errorf("end error %v: Reserve returned %d blocks, error %v", endErr, len(blocks), err) } if got := region.FreeCount(); got != want { t.Errorf("end error %v: %d free blocks after the failed reserve, want %d", endErr, got, want) diff --git a/internal/journal/journal.go b/internal/journal/journal.go index facbd1e0..956db976 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -563,26 +563,53 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { return j.appendCompletionLocked(intentSeq) } -// CompleteIntentDurable returns only after the completion record is synced, -// and keeps the intent pending on error, so a retry writes the record. +// CompleteIntentDurable returns only after the intent's end is synced, and keeps +// the intent pending on error, so a retry ends it. When the intent is the oldest +// and the ring is too full for a completion record, the head moves past it instead. func (j *Journal) CompleteIntentDurable(intentSeq uint64) error { j.mu.Lock() - _, pending := j.pendingIntents[intentSeq] + start, pending := j.pendingIntents[intentSeq] var err error if pending { err = j.appendCompletionLocked(intentSeq) + if errors.Is(err, ErrJournalFull) && j.oldestIntentLocked(intentSeq) { + delete(j.pendingIntents, intentSeq) + if _, err = j.advanceHead(); err != nil { + j.pendingIntents[intentSeq] = start + } + } } j.mu.Unlock() if !pending || err != nil { return err } if err := j.region.Sync(); err != nil { - return fmt.Errorf("failed to sync intent completion: %w", err) + j.mu.Lock() + j.pendingIntents[intentSeq] = start + j.mu.Unlock() + return fmt.Errorf("failed to sync intent end: %w", err) } j.dropIntent(intentSeq) return nil } +func (j *Journal) oldestIntentLocked(intentSeq uint64) bool { + head, size := j.region.HeadOffset(), j.region.DataAreaSize() + after := func(pos uint32) uint32 { + if pos >= head { + return pos - head + } + return pos + size - head + } + mine := after(j.pendingIntents[intentSeq]) + for seq, pos := range j.pendingIntents { + if seq != intentSeq && after(pos) < mine { + return false + } + } + return true +} + func (j *Journal) appendCompletionLocked(intentSeq uint64) error { need := sectorAligned(completionLength) if need > j.region.AvailableSpace() { diff --git a/internal/journal/reserve_test.go b/internal/journal/reserve_test.go index ef0ef371..254ac3c4 100644 --- a/internal/journal/reserve_test.go +++ b/internal/journal/reserve_test.go @@ -205,3 +205,44 @@ func TestCompleteIntentDurableRetriesAfterFailure(t *testing.T) { t.Errorf("replayed %v after a crash, want nothing", got) } } + +// A reserve record that is the oldest intent in a ring too full for a completion +// record still ends: the head moves past it, replay no longer frees its blocks, +// and commits work again. +func TestCompleteIntentDurableEndsOldestIntentInFullRing(t *testing.T) { + r := newReserveJournal(t) + j := r.j + reserveSeq := r.reserve(false, 100, 101) + var others []uint64 + for i := 0; ; i++ { + txn := j.Begin() + if err := txn.LogBlockUse([]uint64{uint64(200 + i)}); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if errors.Is(err, ErrJournalFull) { + j.Abort(txn) + break + } + if err != nil { + t.Fatal(err) + } + others = append(others, seq) + } + for _, s := range others { + _ = j.CompleteIntent(s) + } + if err := j.CompleteIntentDurable(reserveSeq); err != nil { + t.Fatalf("ending the record that pins a full ring: %v", err) + } + if got := r.unused(); len(got) != 0 { + t.Errorf("replay would free %v of the ended record", got) + } + txn := j.Begin() + if err := txn.LogBlockUse([]uint64{1}); err != nil { + t.Fatal(err) + } + if _, err := j.Commit(txn); err != nil { + t.Errorf("commit after ending the record: %v", err) + } +} diff --git a/internal/reserve/reserve.go b/internal/reserve/reserve.go index eeba1765..139efcd7 100644 --- a/internal/reserve/reserve.go +++ b/internal/reserve/reserve.go @@ -172,9 +172,14 @@ func (m *Manager) Untake(blocks ...uint64) { } // Logged runs log, which commits the use of blocks. They leave the reserve even -// if log fails, since a failed commit may still be on disk. +// if log fails, since a failed commit may still be on disk, unless the journal +// was full and wrote nothing. func (m *Manager) Logged(blocks []uint64, log func() error) error { err := log() + if errors.Is(err, journal.ErrJournalFull) { + m.Untake(blocks...) + return err + } m.mu.Lock() defer m.mu.Unlock() for _, b := range blocks { @@ -195,13 +200,21 @@ func (m *Manager) Logged(blocks []uint64, log func() error) error { // ReturnAll gives the untaken blocks back. Each record ends before its blocks are // freed, so a crash in between leaks them instead of replay freeing them twice. +// The frees run after endMu is dropped, so its holders wait only for the journal. func (m *Manager) ReturnAll(ctx context.Context) { + freed, ended := m.endForReturn() + m.free(ctx, freed) + if ended { + m.returns.Add(1) + } +} + +func (m *Manager) endForReturn() (freed []uint64, ended bool) { m.endMu.Lock() defer m.endMu.Unlock() - ctx = context.WithoutCancel(ctx) m.endUsed() if !m.endStale() { - return + return nil, false } groups := m.takeUntaken(func(*record) bool { return true }, false) for r, blocks := range groups { @@ -211,15 +224,9 @@ func (m *Manager) ReturnAll(ctx context.Context) { continue } m.finishEnd(r) - if len(blocks) > 0 { - m.release(ctx, blocks) - m.freeCount.Add(uint64(len(blocks))) - } + freed, ended = append(freed, blocks...), true } - if len(groups) > 0 { - m.returns.Add(1) - } - m.freeOrphans(ctx) + return append(freed, m.takeOrphans()...), ended } // Refresh moves the untaken blocks of records far behind the journal tail into a @@ -277,7 +284,7 @@ func (m *Manager) WaitForPeers(ctx context.Context, waited time.Duration) bool { } if waited == 0 { log.Printf("[INFO] [reserve] the bitmap is full; asking other nodes to give back their reserved blocks") - m.ReturnAll(ctx) + m.AskReturn() } if now := time.Now(); m.broadcast != nil && now.Sub(time.Unix(0, m.askedAt.Load())) >= tick { m.askedAt.Store(now.UnixNano()) @@ -390,7 +397,7 @@ func (m *Manager) refill(ctx context.Context) bool { return false } m.add(seq, blocks) - m.freeCount.Add(^uint64(len(blocks) - 1)) + m.adjustFree(-len(blocks)) m.refills.Add(1) return true } @@ -455,6 +462,10 @@ func (m *Manager) reopen(r *record, blocks []uint64) { m.untaken = append(m.untaken, r.back...) r.back = nil slices.Sort(m.untaken) + if r.live == 0 { + r.state = ending + m.used = append(m.used, r) + } } func (m *Manager) finishEnd(r *record) { @@ -509,25 +520,41 @@ func (m *Manager) endStale() bool { return len(m.stale) == 0 } -func (m *Manager) freeOrphans(ctx context.Context) { +func (m *Manager) takeOrphans() []uint64 { m.mu.Lock() - if len(m.stale) > 0 || len(m.orphans) == 0 { - m.mu.Unlock() - return + defer m.mu.Unlock() + if len(m.stale) > 0 { + return nil } orphans := m.orphans m.orphans = nil - m.mu.Unlock() - m.release(ctx, orphans) - m.freeCount.Add(uint64(len(orphans))) + return orphans +} + +func (m *Manager) free(ctx context.Context, blocks []uint64) { + if len(blocks) == 0 { + return + } + m.release(context.WithoutCancel(ctx), blocks) + m.adjustFree(len(blocks)) +} + +func (m *Manager) adjustFree(delta int) { + for { + old := m.freeCount.Load() + if m.freeCount.CompareAndSwap(old, uint64(max(0, int64(old)+int64(delta)))) { + return + } + } } func (m *Manager) tidy(ctx context.Context) { m.endMu.Lock() - defer m.endMu.Unlock() m.endUsed() m.endStale() - m.freeOrphans(ctx) + orphans := m.takeOrphans() + m.endMu.Unlock() + m.free(ctx, orphans) } func (m *Manager) idle() bool { diff --git a/internal/reserve/reserve_test.go b/internal/reserve/reserve_test.go index 97c42f69..187e10e9 100644 --- a/internal/reserve/reserve_test.go +++ b/internal/reserve/reserve_test.go @@ -241,6 +241,110 @@ func TestFailedLogLeavesBlockAllocated(t *testing.T) { } } +// A commit that hit a full journal wrote nothing, so its blocks go back to the reserve. +func TestJournalFullLogPutsBlockBack(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + b := take(t, m) + if err := m.Logged([]uint64{b}, func() error { return journal.ErrJournalFull }); !errors.Is(err, journal.ErrJournalFull) { + t.Fatalf("Logged returned %v, want the journal-full error", err) + } + if got := m.Stats().Blocks; got != 8 { + t.Errorf("%d blocks in reserve, want 8", got) + } +} + +// The cached free count stops at zero instead of wrapping when a refill +// subtracts blocks a fresh count already left out. +func TestFreeCountNeverWraps(t *testing.T) { + m := newManager(8, newFakeBitmap(10000, 0), &fakeJournal{}) + m.freeCount.Store(3) + m.adjustFree(-8) + if got := m.freeCount.Load(); got != 0 { + t.Errorf("free count %d after subtracting more than it held, want 0", got) + } +} + +// hookJournal runs onCommit before each commit. +type hookJournal struct { + *fakeJournal + onCommit func() error +} + +func (h *hookJournal) Commit(txn *journal.Transaction) (uint64, error) { + if h.onCommit != nil { + if err := h.onCommit(); err != nil { + return 0, err + } + } + return h.fakeJournal.Commit(txn) +} + +// A refresh that fails after a write used the last blocks of a record it was +// ending reopens the record, which then ends as fully used. +func TestRecordReopenedAfterFailedRefreshEnds(t *testing.T) { + ctx := context.Background() + bm, fj := newFakeBitmap(1000, 0), &fakeJournal{} + j := &hookJournal{fakeJournal: fj} + m := New(Config{Size: 2}, bm, j, func(_ context.Context, blocks []uint64) { bm.give(blocks) }, func() int { return 1 }) + if !m.refill(ctx) { + t.Fatal("first refill failed") + } + a1, _ := m.TakeReady() + a2, _ := m.TakeReady() + seqA := fj.seq + if !m.refill(ctx) { + t.Fatal("second refill failed") + } + fj.set(func(j *fakeJournal) { j.lag = 500 }) + j.onCommit = func() error { + j.onCommit = nil + if err := m.Logged([]uint64{a1, a2}, func() error { return nil }); err != nil { + t.Fatal(err) + } + return journal.ErrJournalFull + } + m.Refresh(ctx) + fj.set(func(j *fakeJournal) { j.lag = 0 }) + m.tidy(ctx) + if _, _, pending := fj.IntentLag(seqA); pending { + t.Fatalf("record %d, all of whose blocks were used, never ends", seqA) + } +} + +// An allocation waiting for peers' reserves never waits for endMu, and record +// ends never wait for frees: a holder of either may be stuck on the bitmap lock. +func TestWaitForPeersAndEndsDoNotWaitForFrees(t *testing.T) { + bm, j := newFakeBitmap(10000, 0), &fakeJournal{} + m := newManager(8, bm, j) + m.Untake(take(t, m)) + + j.freeingBlock = make(chan struct{}) + returned := make(chan struct{}) + go func() { + defer close(returned) + m.ReturnAll(context.Background()) + }() + time.Sleep(50 * time.Millisecond) + + done := make(chan struct{}) + go func() { + defer close(done) + m.tidy(context.Background()) + m.Refresh(context.Background()) + m.endMu.Lock() + m.WaitForPeers(context.Background(), 0) + m.endMu.Unlock() + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("a record end or WaitForPeers waited for a free") + } + close(j.freeingBlock) + <-returned +} + // When ending a record fails, its blocks stay reserved, and a later return ends it and frees them. func TestFailedEndKeepsBlocks(t *testing.T) { bm, j := newFakeBitmap(10000, 0), &fakeJournal{} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 8bf5485b..10aeed65 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3199,11 +3199,46 @@ scenarios: - id: bitmap-reserve-restores-on-write-failure name: Failed reserve sector write ends the record, then restores - description: A reserve whose second sector write fails ends its record and restores the first sector; if the record cannot end, the sectors stay as they are + description: A reserve whose second sector write fails ends its record and restores the first sector; if the record cannot end, it writes the failed sector again and keeps the blocks priority: go_only environments: go: internal/bitmap/bitmap_test.go::TestReserveRestoresSectorsWhenAWriteFails + - id: journal-durable-end-full-ring + name: Durable end of the oldest intent in a full ring + description: A reserve record that pins a ring too full for a completion record still ends, because the head moves past it; replay then frees none of its blocks and commits work again + priority: go_only + environments: + go: internal/journal/reserve_test.go::TestCompleteIntentDurableEndsOldestIntentInFullRing + + - id: reserve-journal-full-log + name: Journal-full use commit puts blocks back + description: A use commit that hit a full journal wrote nothing, so its blocks go back to the reserve + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestJournalFullLogPutsBlockBack + + - id: reserve-free-count-saturates + name: Cached free count never wraps + description: Subtracting a refill from a free count that already left it out stops at zero + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestFreeCountNeverWraps + + - id: reserve-reopened-record-ends + name: Record reopened after a failed refresh ends + description: A refresh that fails after a write used a record's last blocks reopens the record, which then ends as fully used + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestRecordReopenedAfterFailedRefreshEnds + + - id: reserve-ends-skip-frees + name: Record ends and peer waits never wait for frees + description: While a return's free waits for the bitmap lock, tidy, refresh and WaitForPeers (even with endMu held) all finish + priority: go_only + environments: + go: internal/reserve/reserve_test.go::TestWaitForPeersAndEndsDoNotWaitForFrees + - id: journal-durable-intent-completion name: Durable intent completion survives a failed write description: CompleteIntentDurable keeps the intent pending when the completion cannot be written, so a retry writes it and replay skips the intent From 646cc7c9c90036a7c23b6ef5a15a21c6e0493e85 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 05:09:36 -0600 Subject: [PATCH 7/7] fix(journal): never put an intent back behind a moved head Third review of #142: if ending the oldest intent in a full ring moved the in-memory head but the superblock write or the sync failed, CompleteIntentDurable put the intent back as pending behind the head. The next checkpoint then wrote a LastCheckpointSeq the head no longer matched, and replay skipped every pending intent. Once the head has moved, the intent now stays out of the pending set and is marked as an unsynced end; the retry rewrites the superblock and syncs before it reports the record ended. Co-Authored-By: Claude Opus 5.5 --- internal/journal/journal.go | 28 +++++++++++---- internal/journal/reserve_test.go | 61 ++++++++++++++++++++++++++++++++ tests/test-manifest.yaml | 7 ++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/internal/journal/journal.go b/internal/journal/journal.go index 956db976..cc87443e 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -1,6 +1,7 @@ package journal import ( + "cmp" "errors" "fmt" "log" @@ -93,6 +94,7 @@ type Journal struct { activeTxns map[*Transaction]*ActiveTxn pendingIntents map[uint64]uint32 // committed but not yet completed, by seq, with where each starts in the ring + unsyncedEnds map[uint64]bool // intents the in-memory head passed before the superblock was synced checkpointHook atomic.Pointer[func()] hookRunning atomic.Bool @@ -138,6 +140,7 @@ func Open(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID config: config, activeTxns: make(map[*Transaction]*ActiveTxn), pendingIntents: make(map[uint64]uint32), + unsyncedEnds: make(map[uint64]bool), checkpointStop: make(chan struct{}), lastCheckpointSeq: region.Superblock().LastCheckpointSeq, } @@ -160,6 +163,7 @@ func Init(dev *storage.Device, regionOffset uint64, regionSize uint64, nodeUUID config: config, activeTxns: make(map[*Transaction]*ActiveTxn), pendingIntents: make(map[uint64]uint32), + unsyncedEnds: make(map[uint64]bool), checkpointStop: make(chan struct{}), lastCheckpointSeq: 0, } @@ -569,27 +573,37 @@ func (j *Journal) CompleteIntent(intentSeq uint64) error { func (j *Journal) CompleteIntentDurable(intentSeq uint64) error { j.mu.Lock() start, pending := j.pendingIntents[intentSeq] + unsynced := j.unsyncedEnds[intentSeq] var err error - if pending { + switch { + case unsynced: + err = j.region.FlushSuperblock() + case pending: err = j.appendCompletionLocked(intentSeq) if errors.Is(err, ErrJournalFull) && j.oldestIntentLocked(intentSeq) { + head := j.region.HeadOffset() delete(j.pendingIntents, intentSeq) - if _, err = j.advanceHead(); err != nil { + _, err = j.advanceHead() + // Once the head moved past the intent, putting it back would checkpoint a seq the head no longer matches. + if j.region.HeadOffset() != head { + j.unsyncedEnds[intentSeq], unsynced = true, true + } else { j.pendingIntents[intentSeq] = start + err = cmp.Or(err, ErrJournalFull) } } } j.mu.Unlock() - if !pending || err != nil { + if (!pending && !unsynced) || err != nil { return err } if err := j.region.Sync(); err != nil { - j.mu.Lock() - j.pendingIntents[intentSeq] = start - j.mu.Unlock() return fmt.Errorf("failed to sync intent end: %w", err) } - j.dropIntent(intentSeq) + j.mu.Lock() + delete(j.pendingIntents, intentSeq) + delete(j.unsyncedEnds, intentSeq) + j.mu.Unlock() return nil } diff --git a/internal/journal/reserve_test.go b/internal/journal/reserve_test.go index 254ac3c4..2c6eb69a 100644 --- a/internal/journal/reserve_test.go +++ b/internal/journal/reserve_test.go @@ -246,3 +246,64 @@ func TestCompleteIntentDurableEndsOldestIntentInFullRing(t *testing.T) { t.Errorf("commit after ending the record: %v", err) } } + +// fillRingBehindReserve commits a reserve record and then completed intents +// until the ring is full, so the reserve record pins the head. +func fillRingBehindReserve(t *testing.T, r *reserveJournal) uint64 { + t.Helper() + j := r.j + reserveSeq := r.reserve(false, 100, 101) + for i := 0; ; i++ { + txn := j.Begin() + if err := txn.LogBlockUse([]uint64{uint64(200 + i)}); err != nil { + t.Fatal(err) + } + seq, err := j.Commit(txn) + if errors.Is(err, ErrJournalFull) { + j.Abort(txn) + return reserveSeq + } + if err != nil { + t.Fatal(err) + } + _ = j.CompleteIntent(seq) + } +} + +// When moving the head past a full ring's oldest intent fails after the head +// moved in memory, the intent does not come back as pending, so the next +// checkpoint stays consistent and replay still runs every pending intent. A +// retry then writes the moved head before it reports the record ended. +func TestFailedHeadMoveKeepsReplayWhole(t *testing.T) { + r := newReserveJournal(t) + j := r.j + reserveSeq := fillRingBehindReserve(t, r) + j.region.dev.SetWriteGate(func(int64) error { return errors.New("injected write error") }) + err := j.CompleteIntentDurable(reserveSeq) + j.region.dev.SetWriteGate(func(int64) error { return nil }) + if err == nil { + t.Fatal("ending the record succeeded although no write could reach the disk") + } + + pending := commitIntents(t, j, true, false)[1] + if _, err := j.Checkpoint(); err != nil { + t.Fatal(err) + } + if got := replayedAfterCrash(t, j); !slices.Contains(got, pending) { + t.Fatalf("replay after a crash ran %v and skipped the pending intent %d", got, pending) + } + + if err := j.CompleteIntentDurable(reserveSeq); err != nil { + t.Fatalf("retrying the end: %v", err) + } + onDisk, err := OpenRegionForRecovery(j.region.dev, j.region.regionOffset, j.region.regionSize) + if err != nil { + t.Fatal(err) + } + if got, want := onDisk.Superblock().HeadOffset, j.region.HeadOffset(); got != want { + t.Errorf("head on disk %d after the retried end, want %d", got, want) + } + if got := r.unused(); len(got) != 0 { + t.Errorf("replay would free %v of the ended record", got) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 10aeed65..03e18fb5 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -3211,6 +3211,13 @@ scenarios: environments: go: internal/journal/reserve_test.go::TestCompleteIntentDurableEndsOldestIntentInFullRing + - id: journal-failed-head-move + name: Failed head move keeps replay whole + description: When moving the head past a full ring's oldest intent fails after the head moved in memory, the intent stays ended, the next checkpoint stays consistent and replay runs every pending intent; a retry writes the moved head + priority: go_only + environments: + go: internal/journal/reserve_test.go::TestFailedHeadMoveKeepsReplayWhole + - id: reserve-journal-full-log name: Journal-full use commit puts blocks back description: A use commit that hit a full journal wrote nothing, so its blocks go back to the reserve