Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ fusermount3 -u mnt # or Ctrl+C the mo
| `internal/fsck` | The fsck phases |
| `internal/metrics` | Atomic counters, not used by `cmd/mount` yet |
| `internal/api` | HTTP API served by `cmd/mount`: `/health`, `/metrics`, `/api/v1/status`, `/api/v1/stats/...`; loopback only unless `-api-token` is set |
| `internal/stats` | Stats types and collector behind `/api/v1/stats` |
| `internal/stats` | Background sampler that feeds `/api/v1/stats/...`, `/metrics` and the capacity in `/api/v1/status` |
| `internal/version` | Build version, set with `-ldflags -X` by `scripts/dist.sh` |

## On-Disk Layout
Expand Down
64 changes: 25 additions & 39 deletions cmd/mount/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,54 +30,23 @@ import (
"github.com/supporttools/GoClusterFS/internal/node"
"github.com/supporttools/GoClusterFS/internal/refcount"
"github.com/supporttools/GoClusterFS/internal/sfb"
"github.com/supporttools/GoClusterFS/internal/stats"
"github.com/supporttools/GoClusterFS/internal/storage"
"github.com/supporttools/GoClusterFS/internal/superblock"
"github.com/supporttools/GoClusterFS/internal/trim"
"github.com/supporttools/GoClusterFS/internal/version"
)

// fsStatusProvider implements api.StatusProvider by wrapping the filesystem.
type fsStatusProvider struct {
mountPath string
devicePath string
dev *storage.Device
bitmapMgr *bitmap.Region
sampler *stats.Sampler
}

func (p *fsStatusProvider) MountPath() string { return p.mountPath }
func (p *fsStatusProvider) DevicePath() string { return p.devicePath }

func (p *fsStatusProvider) MountState() api.MountState {
// If we get here, the filesystem is mounted and running
return api.MountStateMounted
}

func (p *fsStatusProvider) Capacity() *api.CapacityInfo {
totalBlocks := p.bitmapMgr.TotalBlocks()
freeBlocks := p.bitmapMgr.FreeCount()
usedBlocks := totalBlocks - freeBlocks

const blockSize uint64 = goclusterfs.BlockSize
totalBytes := totalBlocks * blockSize
usedBytes := usedBlocks * blockSize
freeBytes := freeBlocks * blockSize

var usedPercent float64
if totalBlocks > 0 {
usedPercent = float64(usedBlocks) / float64(totalBlocks) * 100
}

return &api.CapacityInfo{
TotalBytes: totalBytes,
UsedBytes: usedBytes,
FreeBytes: freeBytes,
UsedPercent: usedPercent,
TotalBlocks: totalBlocks,
UsedBlocks: usedBlocks,
FreeBlocks: freeBlocks,
BlockSizeBytes: blockSize,
}
}
func (p *fsStatusProvider) MountPath() string { return p.mountPath }
func (p *fsStatusProvider) DevicePath() string { return p.devicePath }
func (p *fsStatusProvider) MountState() api.MountState { return api.MountStateMounted }
func (p *fsStatusProvider) Capacity() *api.CapacityInfo { return p.sampler.Capacity() }

func apiConfig(addr, token, corsList string, rateLimit float64) api.Config {
var origins []string
Expand Down Expand Up @@ -452,23 +421,27 @@ func main() {

// Start API server for management and monitoring
var apiServer *api.Server
var sampler *stats.Sampler
if *apiEnabled {
var err error
apiServer, err = api.NewServer(apiCfg)
if err != nil {
log.Printf("Warning: Failed to create API server: %v", err)
} else {
sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, j, trimMgr, bm, nodeReg))
apiServer.SetStatusProvider(&fsStatusProvider{
mountPath: *mountPoint,
devicePath: *devicePath,
dev: dev,
bitmapMgr: bm,
sampler: sampler,
})
apiServer.SetStatsProvider(sampler)
apiServer.RegisterHealthChecker(api.DefaultHealthChecker())
if err := apiServer.Start(ctx); err != nil {
log.Printf("Warning: Failed to start API server: %v", err)
apiServer.Stop()
apiServer = nil
sampler.Stop()
sampler = nil
} else {
log.Printf("API server listening on %s (auth=%v, cors=%v)",
apiServer.Addr(), apiCfg.AuthToken != "", len(apiCfg.CORSOrigins) > 0)
Expand All @@ -490,6 +463,7 @@ func main() {
if err := apiServer.Stop(); err != nil {
log.Printf("Warning: Failed to stop API server: %v", err)
}
sampler.Stop()
}

// Close filesystem (stops snapshot worker, TRIM, journal, etc.)
Expand Down Expand Up @@ -517,6 +491,7 @@ func main() {
if err := apiServer.Stop(); err != nil {
log.Printf("Warning: Failed to stop API server on exit: %v", err)
}
sampler.Stop()
}

// Clean shutdown - close filesystem (handles snapshot worker, TRIM, journal)
Expand Down Expand Up @@ -562,6 +537,17 @@ func main() {
log.Println("Filesystem unmounted.")
}

func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry) stats.Sources {
src := stats.Sources{Cache: cacheMgr, Locks: lockMgr, Blocks: bm, Nodes: nodeReg, BlockSize: goclusterfs.BlockSize}
if j != nil {
src.Journal = j
}
if trimMgr != nil {
src.Trim = trimMgr
}
return src
}

// adoptLocks takes over a dead node's exclusive locks before its journal is
// replayed and returns the function that frees them after the replay.
func adoptLocks(locks *lock.Manager, owner [16]byte) func() {
Expand Down
35 changes: 28 additions & 7 deletions docs/API-CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ This document provides a comprehensive reference for all configuration options i
## Table of Contents

1. [Command-Line Flags](#command-line-flags)
2. [Cache Configuration](#cache-configuration)
3. [Journal Configuration](#journal-configuration)
4. [TRIM Configuration](#trim-configuration)
5. [Broadcast Configuration](#broadcast-configuration)
6. [Configuration Presets](#configuration-presets)
7. [Tuning Recommendations](#tuning-recommendations)
2. [HTTP API Statistics](#http-api-statistics)
3. [Cache Configuration](#cache-configuration)
4. [Journal Configuration](#journal-configuration)
5. [TRIM Configuration](#trim-configuration)
6. [Broadcast Configuration](#broadcast-configuration)
7. [Configuration Presets](#configuration-presets)
8. [Tuning Recommendations](#tuning-recommendations)

---

Expand Down Expand Up @@ -92,7 +93,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For
| `-api-cors` | string | (empty) | Comma-separated CORS allowed origins, or `*` |
| `-api-rate-limit` | float | `0` (off) | Requests per second per client IP, with bursts of twice that, up to 1,000,000. Over the limit, the API answers 429 |

The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. Mount does not feed statistics to the API yet, so `/api/v1/stats/...` answers 503, `/health` only checks the API itself, and `/metrics` only reports `goclusterfs_up` plus Go runtime and process metrics. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer <token>`. `/health` and `/metrics` never need the token.
The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. [HTTP API Statistics](#http-api-statistics) lists what the last three return. `/health` only checks the API itself. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer <token>`. `/health` and `/metrics` never need the token.

Without `-api-token`, the API answers only requests whose `Host` header is `localhost`, a `127.x` address or `[::1]`, so a web page cannot reach it through DNS rebinding, and mount refuses `-api-cors '*'`. List the dashboard's origin instead, for example `-api-cors http://localhost:5173`.

Expand All @@ -104,6 +105,26 @@ Each mount on a host needs its own `-api-addr`. When the port is taken, mount lo

---

## HTTP API Statistics

Mount copies its managers' counters every second and reads the block bitmap and node registry from the device every 15 seconds. Requests get the latest copy, so `/metrics` and `/api/v1/...` never read the device. Capacity and cluster data can be 15 seconds old; `df` reads the device on every call. Counters start at zero when the mount starts and count this node only. Times are RFC 3339 in UTC.

| Endpoint | Returns |
|----------|---------|
| `/api/v1/status` | Version, uptime, mount and device paths, and `capacity`: total, used and free blocks and bytes of the data region, for the whole volume |
| `/api/v1/stats` | `cache` and `journal`, as below |
| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `hits`, `misses` and `hit_rate` (0 to 1) |
| `/api/v1/stats/journal` | This node's journal: `state` (`clean` or `dirty`), `head_offset` and `tail_offset` in the ring, `available_space` in bytes, `active_txn_count`, `commit_count`, `bytes_written`, `last_checkpoint_seq`, `batch_sync_count`. 404 with `-journal=false` |
| `/api/v1/stats/trim` | `trim_enabled`. With `-trim`, also `trim_operations` (discard calls), `trim_blocks` and `trim_bytes` discarded, `pending_trim_blocks`, `trim_errors` (failed ranges), `last_trim_at` and `batch_size`. On a device or image file that cannot discard, ranges still count as discarded |
| `/api/v1/stats/lock` | `active_locks` (locks this node holds), `pending_locks` (lock requests still waiting), `hint_cache_hits`, `hint_cache_misses` and `hint_cache_hit_rate` |
| `/api/v1/stats/cluster` | This node's `node_id` (UUID in hex), `node_address` (hostname), `state`, `last_heartbeat_at` and `joined_at`; `cluster_size` (registered nodes); `healthy_nodes` (active with a heartbeat under 10 seconds old); and `peers`, each with `node_id`, `address` (hostname), `state` and `last_seen`. A state is `active`, `leaving`, `dead`, or `stale` when the heartbeat is over 10 seconds old and no node has declared it dead yet. `role` is always `peer`: there is no leader |
| `/api/v1/stats/sfb` | 404, because SFB containers are disabled |
| `/api/v1/stats/all` | `collected_at` and every section above except `sfb` |

`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*`, `journal_*`, `trim_*`, `lock_*`, `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics.

---

## Cache Configuration

The cache subsystem is configured via the `cache.Config` struct in `internal/cache/config.go`.
Expand Down
35 changes: 30 additions & 5 deletions internal/api/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,13 @@ type fsCollector struct {
clusterSizeDesc *prometheus.Desc
clusterHealthyNodesDesc *prometheus.Desc
clusterIsLeaderDesc *prometheus.Desc

fsSizeBytesDesc *prometheus.Desc
fsFreeBytesDesc *prometheus.Desc
}

var journalStates = map[string]float64{"clean": 0, "dirty": 1, "recovering": 2, "corrupt": 3}

func newFSCollector(s *Server) *fsCollector {
return &fsCollector{
server: s,
Expand Down Expand Up @@ -201,7 +206,7 @@ func newFSCollector(s *Server) *fsCollector {
// Journal
journalStateDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "journal", "state"),
"Journal state (1=open, 0=closed)",
"Journal state: 0=clean, 1=dirty, 2=recovering, 3=corrupt, -1=unknown",
nil, nil,
),
journalHeadOffsetDesc: prometheus.NewDesc(
Expand Down Expand Up @@ -367,6 +372,17 @@ func newFSCollector(s *Server) *fsCollector {
"Whether this node is the cluster leader (1) or not (0)",
nil, nil,
),

fsSizeBytesDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "fs", "size_bytes"),
"Size of the data region in bytes",
nil, nil,
),
fsFreeBytesDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "fs", "free_bytes"),
"Free bytes in the data region",
nil, nil,
),
}
}

Expand Down Expand Up @@ -434,6 +450,9 @@ func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.clusterSizeDesc
ch <- c.clusterHealthyNodesDesc
ch <- c.clusterIsLeaderDesc

ch <- c.fsSizeBytesDesc
ch <- c.fsFreeBytesDesc
}

// Collect is called by the Prometheus registry when collecting metrics.
Expand All @@ -442,6 +461,13 @@ func (c *fsCollector) Collect(ch chan<- prometheus.Metric) {
// Always report up metric
ch <- prometheus.MustNewConstMetric(c.upDesc, prometheus.GaugeValue, 1)

if status := c.server.GetStatusProvider(); status != nil {
if capacity := status.Capacity(); capacity != nil {
ch <- prometheus.MustNewConstMetric(c.fsSizeBytesDesc, prometheus.GaugeValue, float64(capacity.TotalBytes))
ch <- prometheus.MustNewConstMetric(c.fsFreeBytesDesc, prometheus.GaugeValue, float64(capacity.FreeBytes))
}
}

// Get stats provider
provider := c.server.GetStatsProvider()
if provider == nil {
Expand Down Expand Up @@ -506,10 +532,9 @@ func (c *fsCollector) collectCacheStats(ch chan<- prometheus.Metric, stats *Cach
}

func (c *fsCollector) collectJournalStats(ch chan<- prometheus.Metric, stats *JournalStatsResponse) {
// Convert state to numeric (1=open, 0=other)
stateValue := 0.0
if stats.State == "open" {
stateValue = 1.0
stateValue, ok := journalStates[stats.State]
if !ok {
stateValue = -1
}
ch <- prometheus.MustNewConstMetric(c.journalStateDesc, prometheus.GaugeValue, stateValue)
ch <- prometheus.MustNewConstMetric(c.journalHeadOffsetDesc, prometheus.GaugeValue, float64(stats.HeadOffset))
Expand Down
37 changes: 31 additions & 6 deletions internal/api/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) {
},
},
journal: &JournalStatsResponse{
State: "open",
State: "dirty",
HeadOffset: 4096,
TailOffset: 2048,
AvailableSpace: 1048576,
Expand Down Expand Up @@ -210,17 +210,16 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) {
}
}

func TestMetricsEndpoint_JournalClosedState(t *testing.T) {
func TestMetricsEndpoint_JournalCleanState(t *testing.T) {
cfg := DefaultConfig().WithListenAddr(":0")
server, err := NewServer(cfg)
if err != nil {
t.Fatalf("NewServer error: %v", err)
}

// Set up mock provider with closed journal
server.SetStatsProvider(&mockStatsProvider{
journal: &JournalStatsResponse{
State: "closed",
State: "clean",
},
})

Expand All @@ -243,9 +242,35 @@ func TestMetricsEndpoint_JournalClosedState(t *testing.T) {

content := string(body)

// Journal state should be 0 for closed
if !strings.Contains(content, "goclusterfs_journal_state 0") {
t.Error("expected journal_state to be 0 for closed state")
t.Error("expected journal_state to be 0 for a clean journal")
}
}

func TestMetricsEndpoint_ExportsCapacity(t *testing.T) {
server, err := NewServer(DefaultConfig().WithListenAddr("127.0.0.1:0"))
if err != nil {
t.Fatalf("NewServer error: %v", err)
}
server.SetStatusProvider(&mockStatusProvider{capacity: &CapacityInfo{TotalBytes: 3 << 20, FreeBytes: 1 << 20}})
if err := server.Start(context.Background()); err != nil {
t.Fatalf("Start error: %v", err)
}
defer server.Stop()

resp, err := http.Get("http://" + server.Addr() + "/metrics")
if err != nil {
t.Fatalf("request error: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read body error: %v", err)
}
for _, want := range []string{"goclusterfs_fs_size_bytes 3.145728e+06", "goclusterfs_fs_free_bytes 1.048576e+06"} {
if !strings.Contains(string(body), want) {
t.Errorf("/metrics lacks %q", want)
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion internal/api/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ func (s *Server) handleStatsSFB(w http.ResponseWriter, r *http.Request) {

stats := provider.SFBStats()
if stats == nil {
writeError(w, http.StatusNotFound, "sfb stats not available", "block allocator not initialized")
writeError(w, http.StatusNotFound, "sfb stats not available", "SFB containers are disabled")
return
}

Expand Down
Loading
Loading