Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
- Experimental: file snapshots and writable clones through `user.snapshot*` and `user.clone` extended attributes. Read a snapshot by cloning it with `user.snapshot.clone`.
- `goclusterfs-fsck` consistency checker. `-repair` rebuilds block refcounts from every file, snapshot and clone. It refuses to run while the device is in use, and `-repair -dry-run` shows what it would change. Exit codes follow fsck(8): `0` no errors, `1` repaired, `4` errors remain, `8` failed or refused, `16` usage error.
- Benchmark suite (`tests/benchmark`) and an `internal/metrics` package.
- Per-node block reserves (`-block-reserve`, default 64): each node takes a batch of free blocks under one hold of the bitmap lock and hands them to its writes without it, gives unused ones back when idle, at unmount and when space runs low, and waits up to 12 seconds for other nodes' reserves before reporting `ENOSPC`. Journal replay and fsck return the unused reserve of a crashed node. See OPERATIONS.md.
- `/metrics` and `/api/v1/stats/lock` report, per lock type, the acquires served, the acquires that found the lock taken, and how long those waited (`goclusterfs_lock_acquires_total`, `goclusterfs_lock_blocked_total`, `goclusterfs_lock_wait_seconds_total`).
- Go, Docker, and QEMU VM test suites, with `tests/test-manifest.yaml` and `make test-parity` to track coverage across them.
- Fuzz tests for the on-disk decoders (superblock, inode, snapshot descriptor, journal superblock and records) and the journal recovery scan.
Expand Down
23 changes: 21 additions & 2 deletions cmd/goclusterfs-mount/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/supporttools/GoClusterFS/internal/logging"
"github.com/supporttools/GoClusterFS/internal/node"
"github.com/supporttools/GoClusterFS/internal/refcount"
"github.com/supporttools/GoClusterFS/internal/reserve"
"github.com/supporttools/GoClusterFS/internal/stats"
"github.com/supporttools/GoClusterFS/internal/storage"
"github.com/supporttools/GoClusterFS/internal/superblock"
Expand Down Expand Up @@ -111,6 +112,7 @@ func main() {
// TRIM configuration
trimEnabled = flag.Bool("trim", true, "Discard freed blocks on the device (BLKDISCARD) in batches; does nothing on image files")
trimBatchSize = flag.Int("trim-batch", 256, "Freed blocks to collect before a discard")
blockReserve = flag.Int("block-reserve", reserve.DefaultSize, "Data blocks each node takes from the bitmap at once and hands out without the cluster-wide bitmap lock; 0 turns reserves off. Needs the journal")
trimMaxAge = flag.Duration("trim-age", 5*time.Second, "Longest a freed block waits for its discard")

// SFB configuration (Sub-File Block containers for small files)
Expand Down Expand Up @@ -434,6 +436,20 @@ func main() {

// Create filesystem with journal, TRIM, and refcount for crash consistency
gfs := goclusterfs.NewWithComponents(dev, bm, inodeTbl, extentMgr, dirMgr, cacheMgr, j, trimMgr, nil, refcountReg, lockMgr, nodeUUID, dataOffset)
var res *reserve.Manager
if j != nil && *blockReserve > 0 {
res = reserve.New(reserve.Config{Size: *blockReserve}, bm, j, extentMgr.ReleaseBlocks, func() int {
active, _ := nodeReg.GetActiveNodes()
return len(active)
})
res.SetBroadcast(cacheMgr.BroadcastReturnReserves)
cacheMgr.OnReturnReserves(res.AskReturn)
bm.SetNoSpaceWait(res)
j.SetCheckpointHook(func() { res.Refresh(context.Background()) })
gfs.SetReserve(res)
res.Start()
log.Printf("Block reserve: up to %d blocks per node", *blockReserve)
}

// Set up cross-node recovery handler for failed node journal replay
recoveryCtx := &journal.RecoveryContext{
Expand Down Expand Up @@ -506,7 +522,7 @@ func main() {
if err != nil {
log.Printf("Warning: Failed to create the API server: %v; the mount continues without it", err)
} else {
sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, dirMgr, j, trimMgr, bm, nodeReg))
sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, dirMgr, j, trimMgr, bm, nodeReg, res))
apiServer.SetStatusProvider(&fsStatusProvider{
mountPath: *mountPoint,
devicePath: *devicePath,
Expand Down Expand Up @@ -653,8 +669,11 @@ func usageFailf(format string, args ...any) {
os.Exit(cli.ExitUsage)
}

func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, dirMgr *dir.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry) stats.Sources {
func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, dirMgr *dir.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry, res *reserve.Manager) stats.Sources {
src := stats.Sources{Cache: cacheMgr, Locks: lockMgr, Dirs: dirMgr, Blocks: bm, Nodes: nodeReg, BlockSize: goclusterfs.BlockSize}
if res != nil {
src.Reserve = res
}
if j != nil {
src.Journal = j
}
Expand Down
3 changes: 2 additions & 1 deletion docs/API-CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For
|------|------|---------|-------------|
| `-journal` | bool | `true` | Write-ahead journal for crash consistency. Off, a crash of this node can leave the filesystem inconsistent: testing only |
| `-skip-recovery` | int | `-1` (off) | Discard the journal of registry slot N instead of replaying it, if the mount takes slot N (dangerous, may lose data) |
| `-block-reserve` | int | `64` | Data blocks each node takes from the bitmap at once and hands out to writes without the cluster-wide bitmap lock; `0` turns reserves off. Needs `-journal`. See [Block Reserves](OPERATIONS.md#block-reserves) |

### Network Broadcast Flags

Expand Down Expand Up @@ -166,7 +167,7 @@ Mount copies its managers' counters every second. While requests read capacity o
| `/api/v1/stats/sfb` | 404, because SFB containers are disabled |
| `/api/v1/stats/all` | `collected_at` and every section above except `sfb` |

`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics.
`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics.

---

Expand Down
10 changes: 10 additions & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,16 @@ If a node did stall for 15 seconds or more while mounted (hung-task warnings in

What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks as described under [Node Requirements](#node-requirements).

### Block Reserves

Each node takes up to 64 free blocks from the block bitmap at once (`-block-reserve`), under one hold of the cluster-wide bitmap lock, and hands them to its writes without that lock. Without reserves, every new data block takes the bitmap lock, and with four nodes writing at once about 60% of their time went to waiting for it. Only new data blocks of writes come from the reserve. Extent-tree blocks, directory blocks, copy-on-write copies and `fallocate` still take the bitmap lock.

- **Size.** A reserve holds at most `-block-reserve` blocks, and at most 1/16 of the free blocks per mounted node. There are no reserves while fewer blocks are free than 1% of the device or two reserves per mounted node.
- **Going back.** A node gives its unused reserve back within 11 seconds of its last write, at a clean unmount, and when a write finds free space below that line.
- **Full device.** A node that finds no free block gives back its own reserve, asks the other nodes over broadcast to give back theirs, once a second, and retries for up to 12 seconds before it fails with `ENOSPC`. It logs the wait at INFO, and `goclusterfs_reserve_peer_waits_total` and `goclusterfs_reserve_peer_wait_seconds_total` count it. Without broadcast, other nodes still give back idle reserves within 11 seconds. A node answers at most one request a second.
- **`df`.** A node counts its own unused reserve as free, since its writes can use it. Other nodes' reserves count as used until they give them back, so `df` on two nodes can differ by up to 64 MB per node at the default.
- **Crash.** Each reserve is a journal record. When the node's journal is replayed, by the node at its next mount or by another node, the blocks it reserved and never used go back to the bitmap. fsck reports them as "reserved by node N and never used", which is not an error, and `fsck -repair -force` frees them after it marks the reserve done in that node's journal, so a later replay does not free them again. A crash in a few narrow windows leaks a few blocks instead, and `fsck -repair` reclaims them: between ending a reserve and freeing it, and while a write holds a block whose reserve the node ended early, to give it back or because the journal ring was filling up.

### Handling Node Failure

When a node fails unexpectedly:
Expand Down
44 changes: 44 additions & 0 deletions internal/api/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ type fsCollector struct {
lockBlockedDesc *prometheus.Desc
lockWaitSecondsDesc *prometheus.Desc

reserveBlocksDesc *prometheus.Desc
reserveRefillsDesc *prometheus.Desc
reserveReturnsDesc *prometheus.Desc
reservePeerWaitsDesc *prometheus.Desc
reservePeerWaitSecondsDesc *prometheus.Desc

// SFB (Sparse File Bitmap) metrics
sfbTotalBlocksDesc *prometheus.Desc
sfbAllocatedBlocksDesc *prometheus.Desc
Expand Down Expand Up @@ -395,6 +401,31 @@ func newFSCollector(s *Server) *fsCollector {
"Time blocked lock acquires spent waiting, by lock type",
[]string{"type"}, nil,
),
reserveBlocksDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "reserve", "blocks"),
"Data blocks this node holds in reserve and has not handed out",
nil, nil,
),
reserveRefillsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "reserve", "refills_total"),
"Reserves this node took from the bitmap",
nil, nil,
),
reserveReturnsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "reserve", "returns_total"),
"Reserves this node gave back to the bitmap",
nil, nil,
),
reservePeerWaitsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "reserve", "peer_waits_total"),
"Allocations that found the bitmap full and waited for other nodes to give back reserved blocks",
nil, nil,
),
reservePeerWaitSecondsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "reserve", "peer_wait_seconds_total"),
"Time allocations waited for other nodes' reserved blocks",
nil, nil,
),

// SFB (Sparse File Bitmap)
sfbTotalBlocksDesc: prometheus.NewDesc(
Expand Down Expand Up @@ -535,6 +566,11 @@ func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.lockAcquiresDesc
ch <- c.lockBlockedDesc
ch <- c.lockWaitSecondsDesc
ch <- c.reserveBlocksDesc
ch <- c.reserveRefillsDesc
ch <- c.reserveReturnsDesc
ch <- c.reservePeerWaitsDesc
ch <- c.reservePeerWaitSecondsDesc

// SFB
ch <- c.sfbTotalBlocksDesc
Expand Down Expand Up @@ -594,6 +630,14 @@ func (c *fsCollector) Collect(ch chan<- prometheus.Metric) {
if lock := provider.LockStats(); lock != nil {
c.collectLockStats(ch, lock)
}
if all := provider.AllStats(); all != nil && all.Reserve != nil {
r := all.Reserve
ch <- prometheus.MustNewConstMetric(c.reserveBlocksDesc, prometheus.GaugeValue, float64(r.Blocks))
ch <- prometheus.MustNewConstMetric(c.reserveRefillsDesc, prometheus.CounterValue, float64(r.Refills))
ch <- prometheus.MustNewConstMetric(c.reserveReturnsDesc, prometheus.CounterValue, float64(r.Returns))
ch <- prometheus.MustNewConstMetric(c.reservePeerWaitsDesc, prometheus.CounterValue, float64(r.PeerWaits))
ch <- prometheus.MustNewConstMetric(c.reservePeerWaitSecondsDesc, prometheus.CounterValue, r.PeerWaitSeconds)
}

// Collect SFB stats
if sfb := provider.SFBStats(); sfb != nil {
Expand Down
6 changes: 6 additions & 0 deletions internal/api/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) {

// Set up mock provider with all stats
server.SetStatsProvider(&mockStatsProvider{
all: &AllStatsResponse{Reserve: &ReserveStatsResponse{Blocks: 7, Refills: 3, Returns: 2, PeerWaits: 1, PeerWaitSeconds: 0.5}},
cache: &CacheStatsResponse{
InodeCache: CacheHitRateInfo{
Size: 1000,
Expand Down Expand Up @@ -203,6 +204,11 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) {
`goclusterfs_lock_wait_seconds_total{type="bitmap"} 16.5`,
`goclusterfs_lock_acquires_total{type="refcount"} 40`,
`goclusterfs_lock_blocked_total{type="refcount"} 0`,
"goclusterfs_reserve_blocks 7",
"goclusterfs_reserve_refills_total 3",
"goclusterfs_reserve_returns_total 2",
"goclusterfs_reserve_peer_waits_total 1",
"goclusterfs_reserve_peer_wait_seconds_total 0.5",

// SFB metrics
"goclusterfs_sfb_total_blocks 1000",
Expand Down
10 changes: 10 additions & 0 deletions internal/api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,16 @@ type AllStatsResponse struct {
Lock *LockStatsResponse `json:"lock,omitempty"`
SFB *SFBStatsResponse `json:"sfb,omitempty"`
Cluster *ClusterInfoResponse `json:"cluster,omitempty"`
Reserve *ReserveStatsResponse `json:"reserve,omitempty"`
}

// ReserveStatsResponse counts this node's block reserve since it mounted.
type ReserveStatsResponse struct {
Blocks int `json:"blocks"`
Refills uint64 `json:"refills"`
Returns uint64 `json:"returns"`
PeerWaits uint64 `json:"peer_waits"`
PeerWaitSeconds float64 `json:"peer_wait_seconds"`
}

// StatsOverviewResponse is returned by GET /api/v1/stats.
Expand Down
110 changes: 110 additions & 0 deletions internal/bitmap/bitmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,18 @@ type Region struct {
lockMgr *lock.Manager // For distributed locking (optional)
nodeUUID [16]byte // This node's UUID for locking
trimMgr TrimQueuer // Optional TRIM manager for batched discards
noSpace NoSpaceWait
}

// NoSpaceWait lets Alloc wait for blocks that other nodes hold in reserve.
type NoSpaceWait interface {
WaitForPeers(ctx context.Context, waited time.Duration) bool
Waited(d time.Duration)
}

// SetNoSpaceWait makes Alloc wait on w instead of failing when no block is free.
func (r *Region) SetNoSpaceWait(w NoSpaceWait) {
r.noSpace = w
}

// NewRegion creates a new bitmap region manager.
Expand All @@ -89,6 +101,28 @@ func NewRegion(dev *storage.Device, offset, totalBlocks uint64, lockMgr *lock.Ma
// Alloc allocates the requested number of blocks.
// Returns the allocated block numbers, or an error if not enough free blocks.
func (r *Region) Alloc(ctx context.Context, count int) ([]uint64, error) {
var start time.Time
for {
blocks, err := r.allocOnce(ctx, count)
if !errors.Is(err, ErrNoFreeBlocks) || r.noSpace == nil {
if err == nil && !start.IsZero() {
r.noSpace.Waited(time.Since(start))
}
return blocks, err
}
waited := time.Duration(0)
if start.IsZero() {
start = time.Now()
} else {
waited = time.Since(start)
}
if !r.noSpace.WaitForPeers(ctx, waited) {
return nil, err
}
}
}

func (r *Region) allocOnce(ctx context.Context, count int) ([]uint64, error) {
if count <= 0 {
return nil, nil
}
Expand Down Expand Up @@ -332,6 +366,82 @@ func (r *Region) releaseReserved(ctx context.Context, blocks []uint64) {
}
}

// Reserve picks up to n free blocks under one lock hold and calls record with
// them. Only if record reports keep does it mark them allocated. If a sector
// write then fails, it calls end and restores the sectors it changed; if end
// fails, it writes the rest again, so no block a pending record lists stays free on disk.
func (r *Region) Reserve(ctx context.Context, n int, record func([]uint64) (keep bool), end func() error) ([]uint64, error) {
if r.lockMgr != nil {
if err := r.lockMgr.Acquire(ctx, lock.LockTypeBlockBitmap, 0, lock.LockModeExclusive); err != nil {
return nil, fmt.Errorf("failed to acquire bitmap lock: %w", err)
}
defer r.unlock()
}
r.mu.Lock()
defer r.mu.Unlock()

numSectors := (r.totalBlocks + BitsPerSector - 1) / BitsPerSector
var picked, changed []uint64
sectors, originals := make(map[uint64][]byte), make(map[uint64][]byte)
for sectorIdx := uint64(0); sectorIdx < numSectors && len(picked) < n; sectorIdx++ {
data, err := r.readSector(sectorIdx)
if err != nil {
return nil, fmt.Errorf("failed to read bitmap sector %d: %w", sectorIdx, err)
}
original := slices.Clone(data)
before := len(picked)
for bit := uint64(0); bit < BitsPerSector && len(picked) < n; bit++ {
block := sectorIdx*BitsPerSector + bit
if block >= r.totalBlocks {
break
}
if data[bit/8]&(1<<(bit%8)) == 0 {
data[bit/8] |= 1 << (bit % 8)
picked = append(picked, block)
}
}
if len(picked) > before {
sectors[sectorIdx], originals[sectorIdx] = data, original
changed = append(changed, sectorIdx)
}
}
if len(picked) == 0 {
return nil, ErrNoFreeBlocks
}
if !record(picked) {
return nil, errors.New("reserve not recorded")
}
for i, sectorIdx := range changed {
err := r.writeSector(sectorIdx, sectors[sectorIdx])
if err == nil {
continue
}
if eerr := end(); eerr != nil {
if rerr := r.writeSectors(changed[i:], sectors); rerr != nil {
log.Printf("[ERROR] bitmap: reserved blocks %v may be free on disk while their record is pending: %v, %v", picked, eerr, rerr)
return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err)
}
return picked, nil
}
for _, done := range changed[:i+1] {
if rerr := r.writeSector(done, originals[done]); rerr != nil {
log.Printf("[WARN] bitmap: restoring sector %d after a failed reserve failed, its picked blocks stay allocated: %v", done, rerr)
}
}
return nil, fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err)
}
return picked, nil
}

func (r *Region) writeSectors(idx []uint64, sectors map[uint64][]byte) error {
for _, sectorIdx := range idx {
if err := r.writeSector(sectorIdx, sectors[sectorIdx]); err != nil {
return fmt.Errorf("failed to write bitmap sector %d: %w", sectorIdx, err)
}
}
return nil
}

// HoldAllocated takes the bitmap lock and returns the blocks of blocks that are
// allocated. The lock stays held, so none of them can be freed, until release.
// A block past the end of the volume is an error.
Expand Down
Loading
Loading