diff --git a/CLAUDE.md b/CLAUDE.md index de3fe2f2..712b94b3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ fusermount3 -u mnt # or Ctrl+C the mo | `internal/fsck` | The fsck phases | | `internal/metrics` | Atomic counters, not used by `cmd/mount` yet | | `internal/api` | HTTP API served by `cmd/mount`: `/health`, `/metrics`, `/api/v1/status`, `/api/v1/stats/...`; loopback only unless `-api-token` is set | -| `internal/stats` | Stats types and collector behind `/api/v1/stats` | +| `internal/stats` | Background sampler that feeds `/api/v1/stats/...`, `/metrics` and the capacity in `/api/v1/status` | | `internal/version` | Build version, set with `-ldflags -X` by `scripts/dist.sh` | ## On-Disk Layout diff --git a/cmd/mount/main.go b/cmd/mount/main.go index d16507f0..717ff488 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -30,54 +30,23 @@ import ( "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/refcount" "github.com/supporttools/GoClusterFS/internal/sfb" + "github.com/supporttools/GoClusterFS/internal/stats" "github.com/supporttools/GoClusterFS/internal/storage" "github.com/supporttools/GoClusterFS/internal/superblock" "github.com/supporttools/GoClusterFS/internal/trim" "github.com/supporttools/GoClusterFS/internal/version" ) -// fsStatusProvider implements api.StatusProvider by wrapping the filesystem. type fsStatusProvider struct { mountPath string devicePath string - dev *storage.Device - bitmapMgr *bitmap.Region + sampler *stats.Sampler } -func (p *fsStatusProvider) MountPath() string { return p.mountPath } -func (p *fsStatusProvider) DevicePath() string { return p.devicePath } - -func (p *fsStatusProvider) MountState() api.MountState { - // If we get here, the filesystem is mounted and running - return api.MountStateMounted -} - -func (p *fsStatusProvider) Capacity() *api.CapacityInfo { - totalBlocks := p.bitmapMgr.TotalBlocks() - freeBlocks := p.bitmapMgr.FreeCount() - usedBlocks := totalBlocks - freeBlocks - - const blockSize uint64 = goclusterfs.BlockSize - totalBytes := totalBlocks * blockSize - usedBytes := usedBlocks * blockSize - freeBytes := freeBlocks * blockSize - - var usedPercent float64 - if totalBlocks > 0 { - usedPercent = float64(usedBlocks) / float64(totalBlocks) * 100 - } - - return &api.CapacityInfo{ - TotalBytes: totalBytes, - UsedBytes: usedBytes, - FreeBytes: freeBytes, - UsedPercent: usedPercent, - TotalBlocks: totalBlocks, - UsedBlocks: usedBlocks, - FreeBlocks: freeBlocks, - BlockSizeBytes: blockSize, - } -} +func (p *fsStatusProvider) MountPath() string { return p.mountPath } +func (p *fsStatusProvider) DevicePath() string { return p.devicePath } +func (p *fsStatusProvider) MountState() api.MountState { return api.MountStateMounted } +func (p *fsStatusProvider) Capacity() *api.CapacityInfo { return p.sampler.Capacity() } func apiConfig(addr, token, corsList string, rateLimit float64) api.Config { var origins []string @@ -452,23 +421,27 @@ func main() { // Start API server for management and monitoring var apiServer *api.Server + var sampler *stats.Sampler if *apiEnabled { var err error apiServer, err = api.NewServer(apiCfg) if err != nil { log.Printf("Warning: Failed to create API server: %v", err) } else { + sampler = stats.StartSampler(statsSources(cacheMgr, lockMgr, j, trimMgr, bm, nodeReg)) apiServer.SetStatusProvider(&fsStatusProvider{ mountPath: *mountPoint, devicePath: *devicePath, - dev: dev, - bitmapMgr: bm, + sampler: sampler, }) + apiServer.SetStatsProvider(sampler) apiServer.RegisterHealthChecker(api.DefaultHealthChecker()) if err := apiServer.Start(ctx); err != nil { log.Printf("Warning: Failed to start API server: %v", err) apiServer.Stop() apiServer = nil + sampler.Stop() + sampler = nil } else { log.Printf("API server listening on %s (auth=%v, cors=%v)", apiServer.Addr(), apiCfg.AuthToken != "", len(apiCfg.CORSOrigins) > 0) @@ -490,6 +463,7 @@ func main() { if err := apiServer.Stop(); err != nil { log.Printf("Warning: Failed to stop API server: %v", err) } + sampler.Stop() } // Close filesystem (stops snapshot worker, TRIM, journal, etc.) @@ -517,6 +491,7 @@ func main() { if err := apiServer.Stop(); err != nil { log.Printf("Warning: Failed to stop API server on exit: %v", err) } + sampler.Stop() } // Clean shutdown - close filesystem (handles snapshot worker, TRIM, journal) @@ -562,6 +537,17 @@ func main() { log.Println("Filesystem unmounted.") } +func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, j *journal.Journal, trimMgr *trim.Manager, bm *bitmap.Region, nodeReg *node.Registry) stats.Sources { + src := stats.Sources{Cache: cacheMgr, Locks: lockMgr, Blocks: bm, Nodes: nodeReg, BlockSize: goclusterfs.BlockSize} + if j != nil { + src.Journal = j + } + if trimMgr != nil { + src.Trim = trimMgr + } + return src +} + // adoptLocks takes over a dead node's exclusive locks before its journal is // replayed and returns the function that frees them after the replay. func adoptLocks(locks *lock.Manager, owner [16]byte) func() { diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 2248d3ba..177bb60e 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -5,12 +5,13 @@ This document provides a comprehensive reference for all configuration options i ## Table of Contents 1. [Command-Line Flags](#command-line-flags) -2. [Cache Configuration](#cache-configuration) -3. [Journal Configuration](#journal-configuration) -4. [TRIM Configuration](#trim-configuration) -5. [Broadcast Configuration](#broadcast-configuration) -6. [Configuration Presets](#configuration-presets) -7. [Tuning Recommendations](#tuning-recommendations) +2. [HTTP API Statistics](#http-api-statistics) +3. [Cache Configuration](#cache-configuration) +4. [Journal Configuration](#journal-configuration) +5. [TRIM Configuration](#trim-configuration) +6. [Broadcast Configuration](#broadcast-configuration) +7. [Configuration Presets](#configuration-presets) +8. [Tuning Recommendations](#tuning-recommendations) --- @@ -92,7 +93,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For | `-api-cors` | string | (empty) | Comma-separated CORS allowed origins, or `*` | | `-api-rate-limit` | float | `0` (off) | Requests per second per client IP, with bursts of twice that, up to 1,000,000. Over the limit, the API answers 429 | -The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. Mount does not feed statistics to the API yet, so `/api/v1/stats/...` answers 503, `/health` only checks the API itself, and `/metrics` only reports `goclusterfs_up` plus Go runtime and process metrics. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer `. `/health` and `/metrics` never need the token. +The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. [HTTP API Statistics](#http-api-statistics) lists what the last three return. `/health` only checks the API itself. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer `. `/health` and `/metrics` never need the token. Without `-api-token`, the API answers only requests whose `Host` header is `localhost`, a `127.x` address or `[::1]`, so a web page cannot reach it through DNS rebinding, and mount refuses `-api-cors '*'`. List the dashboard's origin instead, for example `-api-cors http://localhost:5173`. @@ -104,6 +105,26 @@ Each mount on a host needs its own `-api-addr`. When the port is taken, mount lo --- +## HTTP API Statistics + +Mount copies its managers' counters every second and reads the block bitmap and node registry from the device every 15 seconds. Requests get the latest copy, so `/metrics` and `/api/v1/...` never read the device. Capacity and cluster data can be 15 seconds old; `df` reads the device on every call. Counters start at zero when the mount starts and count this node only. Times are RFC 3339 in UTC. + +| Endpoint | Returns | +|----------|---------| +| `/api/v1/status` | Version, uptime, mount and device paths, and `capacity`: total, used and free blocks and bytes of the data region, for the whole volume | +| `/api/v1/stats` | `cache` and `journal`, as below | +| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `hits`, `misses` and `hit_rate` (0 to 1) | +| `/api/v1/stats/journal` | This node's journal: `state` (`clean` or `dirty`), `head_offset` and `tail_offset` in the ring, `available_space` in bytes, `active_txn_count`, `commit_count`, `bytes_written`, `last_checkpoint_seq`, `batch_sync_count`. 404 with `-journal=false` | +| `/api/v1/stats/trim` | `trim_enabled`. With `-trim`, also `trim_operations` (discard calls), `trim_blocks` and `trim_bytes` discarded, `pending_trim_blocks`, `trim_errors` (failed ranges), `last_trim_at` and `batch_size`. On a device or image file that cannot discard, ranges still count as discarded | +| `/api/v1/stats/lock` | `active_locks` (locks this node holds), `pending_locks` (lock requests still waiting), `hint_cache_hits`, `hint_cache_misses` and `hint_cache_hit_rate` | +| `/api/v1/stats/cluster` | This node's `node_id` (UUID in hex), `node_address` (hostname), `state`, `last_heartbeat_at` and `joined_at`; `cluster_size` (registered nodes); `healthy_nodes` (active with a heartbeat under 10 seconds old); and `peers`, each with `node_id`, `address` (hostname), `state` and `last_seen`. A state is `active`, `leaving`, `dead`, or `stale` when the heartbeat is over 10 seconds old and no node has declared it dead yet. `role` is always `peer`: there is no leader | +| `/api/v1/stats/sfb` | 404, because SFB containers are disabled | +| `/api/v1/stats/all` | `collected_at` and every section above except `sfb` | + +`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*`, `journal_*`, `trim_*`, `lock_*`, `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics. + +--- + ## Cache Configuration The cache subsystem is configured via the `cache.Config` struct in `internal/cache/config.go`. diff --git a/internal/api/prometheus.go b/internal/api/prometheus.go index c22a8786..9ab9794e 100644 --- a/internal/api/prometheus.go +++ b/internal/api/prometheus.go @@ -97,8 +97,13 @@ type fsCollector struct { clusterSizeDesc *prometheus.Desc clusterHealthyNodesDesc *prometheus.Desc clusterIsLeaderDesc *prometheus.Desc + + fsSizeBytesDesc *prometheus.Desc + fsFreeBytesDesc *prometheus.Desc } +var journalStates = map[string]float64{"clean": 0, "dirty": 1, "recovering": 2, "corrupt": 3} + func newFSCollector(s *Server) *fsCollector { return &fsCollector{ server: s, @@ -201,7 +206,7 @@ func newFSCollector(s *Server) *fsCollector { // Journal journalStateDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "journal", "state"), - "Journal state (1=open, 0=closed)", + "Journal state: 0=clean, 1=dirty, 2=recovering, 3=corrupt, -1=unknown", nil, nil, ), journalHeadOffsetDesc: prometheus.NewDesc( @@ -367,6 +372,17 @@ func newFSCollector(s *Server) *fsCollector { "Whether this node is the cluster leader (1) or not (0)", nil, nil, ), + + fsSizeBytesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "fs", "size_bytes"), + "Size of the data region in bytes", + nil, nil, + ), + fsFreeBytesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "fs", "free_bytes"), + "Free bytes in the data region", + nil, nil, + ), } } @@ -434,6 +450,9 @@ func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.clusterSizeDesc ch <- c.clusterHealthyNodesDesc ch <- c.clusterIsLeaderDesc + + ch <- c.fsSizeBytesDesc + ch <- c.fsFreeBytesDesc } // Collect is called by the Prometheus registry when collecting metrics. @@ -442,6 +461,13 @@ func (c *fsCollector) Collect(ch chan<- prometheus.Metric) { // Always report up metric ch <- prometheus.MustNewConstMetric(c.upDesc, prometheus.GaugeValue, 1) + if status := c.server.GetStatusProvider(); status != nil { + if capacity := status.Capacity(); capacity != nil { + ch <- prometheus.MustNewConstMetric(c.fsSizeBytesDesc, prometheus.GaugeValue, float64(capacity.TotalBytes)) + ch <- prometheus.MustNewConstMetric(c.fsFreeBytesDesc, prometheus.GaugeValue, float64(capacity.FreeBytes)) + } + } + // Get stats provider provider := c.server.GetStatsProvider() if provider == nil { @@ -506,10 +532,9 @@ func (c *fsCollector) collectCacheStats(ch chan<- prometheus.Metric, stats *Cach } func (c *fsCollector) collectJournalStats(ch chan<- prometheus.Metric, stats *JournalStatsResponse) { - // Convert state to numeric (1=open, 0=other) - stateValue := 0.0 - if stats.State == "open" { - stateValue = 1.0 + stateValue, ok := journalStates[stats.State] + if !ok { + stateValue = -1 } ch <- prometheus.MustNewConstMetric(c.journalStateDesc, prometheus.GaugeValue, stateValue) ch <- prometheus.MustNewConstMetric(c.journalHeadOffsetDesc, prometheus.GaugeValue, float64(stats.HeadOffset)) diff --git a/internal/api/prometheus_test.go b/internal/api/prometheus_test.go index 01503cc5..c1180ff2 100644 --- a/internal/api/prometheus_test.go +++ b/internal/api/prometheus_test.go @@ -84,7 +84,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { }, }, journal: &JournalStatsResponse{ - State: "open", + State: "dirty", HeadOffset: 4096, TailOffset: 2048, AvailableSpace: 1048576, @@ -210,17 +210,16 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { } } -func TestMetricsEndpoint_JournalClosedState(t *testing.T) { +func TestMetricsEndpoint_JournalCleanState(t *testing.T) { cfg := DefaultConfig().WithListenAddr(":0") server, err := NewServer(cfg) if err != nil { t.Fatalf("NewServer error: %v", err) } - // Set up mock provider with closed journal server.SetStatsProvider(&mockStatsProvider{ journal: &JournalStatsResponse{ - State: "closed", + State: "clean", }, }) @@ -243,9 +242,35 @@ func TestMetricsEndpoint_JournalClosedState(t *testing.T) { content := string(body) - // Journal state should be 0 for closed if !strings.Contains(content, "goclusterfs_journal_state 0") { - t.Error("expected journal_state to be 0 for closed state") + t.Error("expected journal_state to be 0 for a clean journal") + } +} + +func TestMetricsEndpoint_ExportsCapacity(t *testing.T) { + server, err := NewServer(DefaultConfig().WithListenAddr("127.0.0.1:0")) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + server.SetStatusProvider(&mockStatusProvider{capacity: &CapacityInfo{TotalBytes: 3 << 20, FreeBytes: 1 << 20}}) + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + for _, want := range []string{"goclusterfs_fs_size_bytes 3.145728e+06", "goclusterfs_fs_free_bytes 1.048576e+06"} { + if !strings.Contains(string(body), want) { + t.Errorf("/metrics lacks %q", want) + } } } diff --git a/internal/api/routes.go b/internal/api/routes.go index 5d510894..6a0ecf51 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -226,7 +226,7 @@ func (s *Server) handleStatsSFB(w http.ResponseWriter, r *http.Request) { stats := provider.SFBStats() if stats == nil { - writeError(w, http.StatusNotFound, "sfb stats not available", "block allocator not initialized") + writeError(w, http.StatusNotFound, "sfb stats not available", "SFB containers are disabled") return } diff --git a/internal/lock/lock.go b/internal/lock/lock.go index f88489b1..063cd18f 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -175,6 +175,12 @@ func (c *hintLRU) Put(key uint64, value *lockHint) { c.items[key] = elem } +func (c *hintLRU) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.order.Len() +} + // Peek returns the value without updating LRU order. func (c *hintLRU) Peek(key uint64) (*lockHint, bool) { c.mu.Lock() @@ -224,6 +230,17 @@ type Manager struct { // Metrics for hint cache effectiveness hintHits uint64 // Optimistic acquires that succeeded hintMisses uint64 // Optimistic acquires that failed (fell back to read) + + waiting atomic.Int64 +} + +// Stats counts this node's lock activity. +type Stats struct { + Held int // lock keys this node holds + Waiting int // Acquire calls not yet granted + HintCacheSize int + HintHits uint64 + HintMisses uint64 } type lockKey struct { @@ -343,6 +360,20 @@ func (m *Manager) HintStats() (hits, misses uint64, hitRate float64) { return } +// Stats returns this node's lock counts without touching the device. +func (m *Manager) Stats() Stats { + m.mu.Lock() + held := len(m.heldLocks) + m.mu.Unlock() + return Stats{ + Held: held, + Waiting: int(m.waiting.Load()), + HintCacheSize: m.hintCache.Len(), + HintHits: atomic.LoadUint64(&m.hintHits), + HintMisses: atomic.LoadUint64(&m.hintMisses), + } +} + // Acquire attempts to acquire a lock with the given parameters. // Blocks until the lock is acquired or context is cancelled. // @@ -369,6 +400,8 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin return err } m.mu.Unlock() + m.waiting.Add(1) + defer m.waiting.Add(-1) retryCount := 0 startTime := time.Now() diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 09eaf89b..49f05a30 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -438,3 +438,39 @@ func TestTimedOutAcquireLogsHolder(t *testing.T) { t.Fatalf("timeout log does not name the holder:\n%s", got) } } + +func TestStatsCountsHeldAndWaitingLocks(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + a := newTestNode(dev, 0, RecordSize, 1) + b := newTestNode(dev, 0, RecordSize, 2) + + requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) + if got := a.Stats(); got.Held != 1 || got.Waiting != 0 { + t.Fatalf("holder Stats = %+v, want Held 1, Waiting 0", got) + } + + granted := make(chan error, 1) + go func() { granted <- acquireWithin(b, 10*time.Second, LockTypeInode, 1, LockModeExclusive) }() + deadline := time.Now().Add(5 * time.Second) + for b.Stats().Waiting != 1 { + if time.Now().After(deadline) { + t.Fatalf("waiter Stats = %+v, want Waiting 1", b.Stats()) + } + time.Sleep(5 * time.Millisecond) + } + + if err := a.Release(LockTypeInode, 1); err != nil { + t.Fatalf("Release: %v", err) + } + if err := <-granted; err != nil { + t.Fatalf("waiter Acquire: %v", err) + } + if got := b.Stats(); got.Held != 1 || got.Waiting != 0 { + t.Errorf("waiter Stats after grant = %+v, want Held 1, Waiting 0", got) + } + if got := a.Stats(); got.Held != 0 { + t.Errorf("holder Stats after release = %+v, want Held 0", got) + } + b.Release(LockTypeInode, 1) +} diff --git a/internal/node/liveness.go b/internal/node/liveness.go index 9ec1c309..dccd8e29 100644 --- a/internal/node/liveness.go +++ b/internal/node/liveness.go @@ -45,13 +45,14 @@ func readPIDNamespace() uint64 { return st.Ino } -func hostnameOf(slot *Slot) string { - return string(bytes.TrimRight(slot.Hostname[:], "\x00")) +// Host returns the hostname the slot's node registered with. +func (s *Slot) Host() string { + return string(bytes.TrimRight(s.Hostname[:], "\x00")) } // sameHost reports whether slot was registered under this node's hostname and kernel boot. func (r *Registry) sameHost(slot *Slot) bool { - return r.bootID != ([16]byte{}) && slot.BootID == r.bootID && hostnameOf(slot) == r.hostname + return r.bootID != ([16]byte{}) && slot.BootID == r.bootID && slot.Host() == r.hostname } // deadLocalOwner reports whether slot belongs to an exited mount process in diff --git a/internal/node/registry.go b/internal/node/registry.go index 35fc2975..eda7d8f9 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -609,11 +609,8 @@ func (r *Registry) GetActiveNodes() ([]*Slot, error) { } // Slots returns every node slot, with nil for empty or unreadable ones. The -// error names the unreadable ones. +// error names the unreadable ones. It holds no lock, so it never delays a heartbeat. func (r *Registry) Slots() ([]*Slot, error) { - r.mu.RLock() - defer r.mu.RUnlock() - slots := make([]*Slot, MaxNodes) var errs []error for i := range slots { @@ -844,7 +841,7 @@ func (r *Registry) acquireForRecovery(slotIdx int, observed *Slot, timeout time. JournalOffset: slot.JournalOffset, JournalSize: slot.JournalSize, LastHeartbeat: time.Unix(slot.HeartbeatTime, 0), - Hostname: hostnameOf(slot), + Hostname: slot.Host(), CheckLease: r.leaseCheck(slotIdx, slot.UUID), }, nil } diff --git a/internal/stats/cache.go b/internal/stats/cache.go deleted file mode 100644 index 10f25979..00000000 --- a/internal/stats/cache.go +++ /dev/null @@ -1,91 +0,0 @@ -package stats - -import "time" - -// CacheInfo contains metadata about the statistics cache state. -type CacheInfo struct { - // Valid indicates if the cache contains valid data. - Valid bool `json:"valid"` - - // Age is how long since the cached data was collected. - Age time.Duration `json:"age"` - - // TTL is the configured cache time-to-live. - TTL time.Duration `json:"ttl"` - - // Stale indicates if the cache is past its TTL (needs refresh). - Stale bool `json:"stale"` - - // LastCollected is when the cached snapshot was collected. - LastCollected time.Time `json:"last_collected,omitempty"` -} - -// CacheInfo returns information about the current cache state. -func (c *Collector) CacheInfo() CacheInfo { - c.mu.RLock() - defer c.mu.RUnlock() - - info := CacheInfo{ - Valid: c.cacheValid, - TTL: c.config.CacheTTL, - } - - if c.cacheValid { - info.Age = time.Since(c.cacheTime) - info.LastCollected = c.cacheTime - info.Stale = info.Age >= c.config.CacheTTL - } - - return info -} - -// IsCacheValid returns true if the cache contains valid, non-stale data. -func (c *Collector) IsCacheValid() bool { - c.mu.RLock() - defer c.mu.RUnlock() - - return c.cacheValid && time.Since(c.cacheTime) < c.config.CacheTTL -} - -// CacheAge returns how long since the cache was last refreshed. -// Returns 0 if the cache is not valid. -func (c *Collector) CacheAge() time.Duration { - c.mu.RLock() - defer c.mu.RUnlock() - - if !c.cacheValid { - return 0 - } - return time.Since(c.cacheTime) -} - -// CachedSnapshot wraps a Snapshot with cache metadata. -// This is useful for API responses that need to indicate cache freshness. -type CachedSnapshot struct { - // Snapshot contains the actual statistics. - *Snapshot - - // FromCache indicates if this snapshot was served from cache. - FromCache bool `json:"from_cache"` - - // CacheAge is how old the cached data is. - CacheAge time.Duration `json:"cache_age,omitempty"` -} - -// CollectWithMeta returns a snapshot along with cache metadata. -func (c *Collector) CollectWithMeta() CachedSnapshot { - c.mu.RLock() - wasValid := c.cacheValid && time.Since(c.cacheTime) < c.config.CacheTTL - c.mu.RUnlock() - - snapshot := c.Collect() - - c.mu.RLock() - defer c.mu.RUnlock() - - return CachedSnapshot{ - Snapshot: snapshot, - FromCache: wasValid, - CacheAge: time.Since(c.cacheTime), - } -} diff --git a/internal/stats/collector.go b/internal/stats/collector.go deleted file mode 100644 index f84984a6..00000000 --- a/internal/stats/collector.go +++ /dev/null @@ -1,292 +0,0 @@ -package stats - -import ( - "sync" - "time" -) - -// Collector aggregates statistics from multiple components. -// It caches collected statistics for a configurable TTL to avoid -// expensive repeated queries. -type Collector struct { - mu sync.RWMutex - config Config - - // Providers - cacheProvider CacheStatsProvider - journalProvider JournalStatsProvider - filesystemProvider FilesystemStatsProvider - lockProvider LockStatsProvider - trimProvider TrimStatsProvider - sfbProvider SFBStatsProvider - clusterProvider ClusterInfoProvider - - // Cached snapshot - cache *Snapshot - cacheTime time.Time - cacheValid bool -} - -// NewCollector creates a new statistics collector with the given configuration. -func NewCollector(cfg Config) *Collector { - return &Collector{ - config: cfg, - } -} - -// SetCacheProvider sets the cache statistics provider. -func (c *Collector) SetCacheProvider(provider CacheStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.cacheProvider = provider - c.invalidateCache() -} - -// SetJournalProvider sets the journal statistics provider. -func (c *Collector) SetJournalProvider(provider JournalStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.journalProvider = provider - c.invalidateCache() -} - -// SetFilesystemProvider sets the filesystem statistics provider. -func (c *Collector) SetFilesystemProvider(provider FilesystemStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.filesystemProvider = provider - c.invalidateCache() -} - -// SetLockProvider sets the lock statistics provider. -func (c *Collector) SetLockProvider(provider LockStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.lockProvider = provider - c.invalidateCache() -} - -// SetTrimProvider sets the TRIM statistics provider. -func (c *Collector) SetTrimProvider(provider TrimStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.trimProvider = provider - c.invalidateCache() -} - -// SetSFBProvider sets the Sparse File Bitmap statistics provider. -func (c *Collector) SetSFBProvider(provider SFBStatsProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.sfbProvider = provider - c.invalidateCache() -} - -// SetClusterProvider sets the cluster information provider. -func (c *Collector) SetClusterProvider(provider ClusterInfoProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.clusterProvider = provider - c.invalidateCache() -} - -// Collect returns a snapshot of all current statistics. -// The snapshot may be cached if within the TTL period. -func (c *Collector) Collect() *Snapshot { - c.mu.RLock() - if c.cacheValid && time.Since(c.cacheTime) < c.config.CacheTTL { - snapshot := c.cache - c.mu.RUnlock() - return snapshot - } - c.mu.RUnlock() - - // Need to collect fresh stats - c.mu.Lock() - defer c.mu.Unlock() - - // Double-check after acquiring write lock - if c.cacheValid && time.Since(c.cacheTime) < c.config.CacheTTL { - return c.cache - } - - // Collect fresh statistics - snapshot := c.collectLocked() - c.cache = snapshot - c.cacheTime = time.Now() - c.cacheValid = true - - return snapshot -} - -// CollectFresh forces collection of fresh statistics, bypassing the cache. -func (c *Collector) CollectFresh() *Snapshot { - c.mu.Lock() - defer c.mu.Unlock() - - snapshot := c.collectLocked() - c.cache = snapshot - c.cacheTime = time.Now() - c.cacheValid = true - - return snapshot -} - -// InvalidateCache forces the next Collect call to fetch fresh data. -func (c *Collector) InvalidateCache() { - c.mu.Lock() - defer c.mu.Unlock() - c.invalidateCache() -} - -// invalidateCache marks the cache as invalid (caller must hold mu). -func (c *Collector) invalidateCache() { - c.cacheValid = false -} - -// collectLocked performs the actual statistics collection. -// Caller must hold c.mu. -func (c *Collector) collectLocked() *Snapshot { - snapshot := &Snapshot{ - CollectedAt: time.Now(), - } - - // Collect from each provider if available - if c.config.EnableComponentStats { - if c.cacheProvider != nil { - snapshot.Cache = c.cacheProvider.CacheStats() - } - - if c.journalProvider != nil { - snapshot.Journal = c.journalProvider.JournalStats() - } - - if c.filesystemProvider != nil { - snapshot.Filesystem = c.filesystemProvider.FilesystemStats() - } - - if c.lockProvider != nil { - snapshot.Lock = c.lockProvider.LockStats() - } - - if c.trimProvider != nil { - snapshot.Trim = c.trimProvider.TrimStats() - } - - if c.sfbProvider != nil { - snapshot.SFB = c.sfbProvider.SFBStats() - } - - if c.clusterProvider != nil { - snapshot.Cluster = c.clusterProvider.ClusterInfo() - } - } - - return snapshot -} - -// CacheStats returns just the cache statistics. -// Returns nil if no cache provider is set. -func (c *Collector) CacheStats() *CacheStats { - c.mu.RLock() - provider := c.cacheProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.CacheStats() -} - -// JournalStats returns just the journal statistics. -// Returns nil if no journal provider is set. -func (c *Collector) JournalStats() *JournalStats { - c.mu.RLock() - provider := c.journalProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.JournalStats() -} - -// FilesystemStats returns just the filesystem statistics. -// Returns nil if no filesystem provider is set. -func (c *Collector) FilesystemStats() *FilesystemStats { - c.mu.RLock() - provider := c.filesystemProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.FilesystemStats() -} - -// LockStats returns just the lock statistics. -// Returns nil if no lock provider is set. -func (c *Collector) LockStats() *LockStats { - c.mu.RLock() - provider := c.lockProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.LockStats() -} - -// TrimStats returns just the TRIM statistics. -// Returns nil if no trim provider is set. -func (c *Collector) TrimStats() *TrimStats { - c.mu.RLock() - provider := c.trimProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.TrimStats() -} - -// SFBStats returns just the Sparse File Bitmap statistics. -// Returns nil if no SFB provider is set. -func (c *Collector) SFBStats() *SFBStats { - c.mu.RLock() - provider := c.sfbProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.SFBStats() -} - -// ClusterInfo returns just the cluster information. -// Returns nil if no cluster provider is set. -func (c *Collector) ClusterInfo() *ClusterInfo { - c.mu.RLock() - provider := c.clusterProvider - c.mu.RUnlock() - - if provider == nil { - return nil - } - return provider.ClusterInfo() -} - -// Config returns the collector's configuration. -func (c *Collector) Config() Config { - c.mu.RLock() - defer c.mu.RUnlock() - return c.config -} - -// SetConfig updates the collector's configuration. -func (c *Collector) SetConfig(cfg Config) { - c.mu.Lock() - defer c.mu.Unlock() - c.config = cfg - c.invalidateCache() -} diff --git a/internal/stats/convert.go b/internal/stats/convert.go new file mode 100644 index 00000000..2187b22e --- /dev/null +++ b/internal/stats/convert.go @@ -0,0 +1,147 @@ +package stats + +import ( + "encoding/hex" + "strings" + "time" + + "github.com/supporttools/GoClusterFS/internal/api" + "github.com/supporttools/GoClusterFS/internal/cache" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/trim" +) + +func hitRate(hits, misses uint64) float64 { + if hits+misses == 0 { + return 0 + } + return float64(hits) / float64(hits+misses) +} + +func hitRateInfo(size int, hits, misses uint64) api.CacheHitRateInfo { + return api.CacheHitRateInfo{Size: size, Hits: hits, Misses: misses, HitRate: hitRate(hits, misses)} +} + +func cacheResponse(c cache.Stats, l lock.Stats) *api.CacheStatsResponse { + return &api.CacheStatsResponse{ + InodeCache: hitRateInfo(c.InodeCacheSize, c.InodeCacheHits, c.InodeCacheMisses), + ExtentCache: hitRateInfo(c.ExtentCacheSize, c.ExtentCacheHits, c.ExtentCacheMisses), + NegativeCache: hitRateInfo(c.NegativeCacheSize, c.NegativeHits, c.NegativeMisses), + LockHintCache: hitRateInfo(l.HintCacheSize, l.HintHits, l.HintMisses), + } +} + +func lockResponse(l lock.Stats) *api.LockStatsResponse { + return &api.LockStatsResponse{ + HintCacheHits: l.HintHits, + HintCacheMisses: l.HintMisses, + HintCacheHitRate: hitRate(l.HintHits, l.HintMisses), + ActiveLocks: l.Held, + PendingLocks: l.Waiting, + } +} + +func journalResponse(j journal.JournalStats) *api.JournalStatsResponse { + return &api.JournalStatsResponse{ + State: strings.ToLower(j.State.String()), + HeadOffset: j.HeadOffset, + TailOffset: j.TailOffset, + AvailableSpace: j.AvailableSpace, + ActiveTxnCount: j.ActiveTxnCount, + CommitCount: j.CommitCount, + BytesWritten: j.BytesWritten, + LastCheckpointSeq: j.LastCheckpointSeq, + BatchSyncCount: j.BatchSyncCount, + } +} + +func trimResponse(t trim.Stats) *api.TrimStatsResponse { + return &api.TrimStatsResponse{ + TrimOperations: t.DiscardCalls, + TrimBytes: t.BlocksDiscarded * trim.BlockSize, + TrimBlocks: t.BlocksDiscarded, + PendingTrimBlocks: t.PendingBlocks, + LastTrimAt: timestamp(t.LastDiscard), + TrimErrors: t.DiscardErrors, + TrimEnabled: true, + BatchSize: t.BatchSize, + } +} + +func capacityInfo(totalBlocks, freeBlocks, blockSize uint64) *api.CapacityInfo { + freeBlocks = min(freeBlocks, totalBlocks) + used := totalBlocks - freeBlocks + info := &api.CapacityInfo{ + TotalBytes: totalBlocks * blockSize, + UsedBytes: used * blockSize, + FreeBytes: freeBlocks * blockSize, + TotalBlocks: totalBlocks, + UsedBlocks: used, + FreeBlocks: freeBlocks, + BlockSizeBytes: blockSize, + } + if totalBlocks > 0 { + info.UsedPercent = float64(used) / float64(totalBlocks) * 100 + } + return info +} + +// clusterResponse describes the registry's non-empty slots. GoClusterFS has no +// leader, so every node reports the role "peer". +func clusterResponse(localUUID [16]byte, localSlot int, slots []*node.Slot, now time.Time) *api.ClusterInfoResponse { + resp := &api.ClusterInfoResponse{ + NodeID: hex.EncodeToString(localUUID[:]), + Role: "peer", + State: "unknown", + } + for i, slot := range slots { + if slot == nil { + continue + } + state := slotState(slot, now) + resp.ClusterSize++ + if state == node.NodeStateActive.String() { + resp.HealthyNodes++ + } + if i == localSlot { + resp.NodeAddress = slot.Host() + resp.State = state + resp.LastHeartbeatAt = timestamp(unixTime(slot.HeartbeatTime)) + resp.JoinedAt = timestamp(unixTime(slot.MountTime)) + continue + } + resp.Peers = append(resp.Peers, api.ClusterPeerInfo{ + NodeID: hex.EncodeToString(slot.UUID[:]), + Address: slot.Host(), + State: state, + LastSeen: timestamp(unixTime(slot.HeartbeatTime)), + }) + } + return resp +} + +// slotState is the slot's registry state, or "stale" for a node whose heartbeat +// is older than node.HeartbeatTimeout but that no peer has declared dead yet. +func slotState(slot *node.Slot, now time.Time) string { + live := slot.State == node.NodeStateActive || slot.State == node.NodeStateLeaving + if live && now.After(slot.LeaseExpiry()) { + return "stale" + } + return slot.State.String() +} + +func unixTime(sec int64) time.Time { + if sec == 0 { + return time.Time{} + } + return time.Unix(sec, 0) +} + +func timestamp(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) +} diff --git a/internal/stats/doc.go b/internal/stats/doc.go index cea2e452..175fddaf 100644 --- a/internal/stats/doc.go +++ b/internal/stats/doc.go @@ -1,48 +1,8 @@ -// Package stats provides unified statistics collection and aggregation for GoClusterFS. +// Package stats feeds the HTTP API's /api/v1/stats endpoints, /metrics and the +// capacity in /api/v1/status from the mount's managers. // -// The stats package serves as an aggregation layer that collects statistics from -// multiple internal components (cache, journal, filesystem, etc.) and presents -// them through a unified interface for API endpoints and monitoring. -// -// # Architecture -// -// Unlike the metrics package which provides raw counters/gauges/histograms, -// the stats package provides high-level system status aggregation: -// -// - Cache stats: hit rates, sizes, broadcast status -// - Journal stats: transaction counts, space utilization -// - Filesystem stats: file counts, capacity, mount state -// - Lock stats: contention metrics, hint cache performance -// -// # Collection Model -// -// Statistics are collected on-demand using a "pull" model. The [Collector] queries -// registered providers when [Collector.Collect] is called. Results are cached -// with a configurable TTL to avoid expensive repeated queries. -// -// # Provider Interfaces -// -// Components implement provider interfaces to expose their statistics: -// -// - [CacheStatsProvider]: cache hit/miss rates and sizes -// - [JournalStatsProvider]: journal state and transaction stats -// - [FilesystemStatsProvider]: file counts and capacity -// - [LockStatsProvider]: lock contention metrics -// -// This interface-based approach keeps the stats package decoupled from -// internal implementation details. -// -// # Usage -// -// // Create collector with 1-second cache TTL -// cfg := stats.DefaultConfig() -// collector := stats.NewCollector(cfg) -// -// // Register providers -// collector.SetCacheProvider(cacheManager) -// collector.SetJournalProvider(journal) -// -// // Collect statistics (cached for TTL) -// snapshot := collector.Collect() -// fmt.Printf("Cache hit rate: %.2f%%\n", snapshot.Cache.InodeHitRate*100) +// A [Sampler] copies each manager's counters every [CounterInterval] and reads +// the block bitmap and node registry from the device every [DeviceInterval]. +// API requests only load the latest snapshot, so an unauthenticated /metrics +// scrape never reads the device or waits on a manager's lock. package stats diff --git a/internal/stats/errors.go b/internal/stats/errors.go deleted file mode 100644 index 6e7268b3..00000000 --- a/internal/stats/errors.go +++ /dev/null @@ -1,12 +0,0 @@ -package stats - -import "errors" - -// Package-level errors for stats collection. -var ( - // ErrInvalidTTL is returned when the cache TTL is invalid. - ErrInvalidTTL = errors.New("stats: invalid cache TTL") - - // ErrCollectorStopped is returned when operations are attempted on a stopped collector. - ErrCollectorStopped = errors.New("stats: collector is stopped") -) diff --git a/internal/stats/sampler.go b/internal/stats/sampler.go new file mode 100644 index 00000000..c882b599 --- /dev/null +++ b/internal/stats/sampler.go @@ -0,0 +1,161 @@ +package stats + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/supporttools/GoClusterFS/internal/api" + "github.com/supporttools/GoClusterFS/internal/cache" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/trim" +) + +const ( + CounterInterval = time.Second + DeviceInterval = 15 * time.Second +) + +type CacheSource interface{ Stats() cache.Stats } + +type LockSource interface{ Stats() lock.Stats } + +type JournalSource interface{ Stats() journal.JournalStats } + +type TrimSource interface{ Stats() trim.Stats } + +type BlockSource interface { + TotalBlocks() uint64 + FreeCount() uint64 +} + +type NodeSource interface { + LocalUUID() [16]byte + LocalSlot() int + Slots() ([]*node.Slot, error) +} + +// Sources are the managers a Sampler reads. Leave Journal or Trim nil when the +// mount runs without them. +type Sources struct { + Cache CacheSource + Locks LockSource + Journal JournalSource + Trim TrimSource + Blocks BlockSource + Nodes NodeSource + BlockSize uint64 +} + +// Sampler implements api.StatsProvider and the capacity half of +// api.StatusProvider from snapshots it refreshes in the background. +type Sampler struct { + src Sources + counterEvery time.Duration + deviceEvery time.Duration + + current api.AllStatsResponse + all atomic.Pointer[api.AllStatsResponse] + capacity atomic.Pointer[api.CapacityInfo] + + stop chan struct{} + done chan struct{} + stopOnce sync.Once +} + +var _ api.StatsProvider = (*Sampler)(nil) + +// StartSampler takes a first sample of every source and keeps refreshing it until Stop. +func StartSampler(src Sources) *Sampler { + return startSampler(src, CounterInterval, DeviceInterval) +} + +func startSampler(src Sources, counterEvery, deviceEvery time.Duration) *Sampler { + s := &Sampler{ + src: src, + counterEvery: counterEvery, + deviceEvery: deviceEvery, + stop: make(chan struct{}), + done: make(chan struct{}), + } + s.sampleDevice() + s.sampleCounters() + go s.run() + return s +} + +// Stop ends sampling and waits for a sample in progress to finish. The last +// snapshot stays readable. +func (s *Sampler) Stop() { + s.stopOnce.Do(func() { close(s.stop) }) + <-s.done +} + +func (s *Sampler) run() { + defer close(s.done) + counters := time.NewTicker(s.counterEvery) + defer counters.Stop() + device := time.NewTicker(s.deviceEvery) + defer device.Stop() + for { + select { + case <-s.stop: + return + case <-counters.C: + s.sampleCounters() + case <-device.C: + s.sampleDevice() + } + } +} + +func (s *Sampler) sampleCounters() { + var locks lock.Stats + if s.src.Locks != nil { + locks = s.src.Locks.Stats() + s.current.Lock = lockResponse(locks) + } + if s.src.Cache != nil { + s.current.Cache = cacheResponse(s.src.Cache.Stats(), locks) + } + if s.src.Journal != nil { + s.current.Journal = journalResponse(s.src.Journal.Stats()) + } + s.current.Trim = &api.TrimStatsResponse{} + if s.src.Trim != nil { + s.current.Trim = trimResponse(s.src.Trim.Stats()) + } + s.current.CollectedAt = time.Now().UTC().Format(time.RFC3339Nano) + s.publish() +} + +func (s *Sampler) sampleDevice() { + if s.src.Blocks != nil { + s.capacity.Store(capacityInfo(s.src.Blocks.TotalBlocks(), s.src.Blocks.FreeCount(), s.src.BlockSize)) + } + if s.src.Nodes != nil { + slots, _ := s.src.Nodes.Slots() + s.current.Cluster = clusterResponse(s.src.Nodes.LocalUUID(), s.src.Nodes.LocalSlot(), slots, time.Now()) + } + s.publish() +} + +func (s *Sampler) publish() { + snapshot := s.current + s.all.Store(&snapshot) +} + +func (s *Sampler) CacheStats() *api.CacheStatsResponse { return s.all.Load().Cache } +func (s *Sampler) JournalStats() *api.JournalStatsResponse { return s.all.Load().Journal } +func (s *Sampler) TrimStats() *api.TrimStatsResponse { return s.all.Load().Trim } +func (s *Sampler) LockStats() *api.LockStatsResponse { return s.all.Load().Lock } +func (s *Sampler) ClusterInfo() *api.ClusterInfoResponse { return s.all.Load().Cluster } +func (s *Sampler) AllStats() *api.AllStatsResponse { return s.all.Load() } + +// SFBStats returns nil because SFB containers are disabled. +func (s *Sampler) SFBStats() *api.SFBStatsResponse { return nil } + +// Capacity returns the block bitmap's counts from the last device sample. +func (s *Sampler) Capacity() *api.CapacityInfo { return s.capacity.Load() } diff --git a/internal/stats/sampler_test.go b/internal/stats/sampler_test.go new file mode 100644 index 00000000..4101dd6f --- /dev/null +++ b/internal/stats/sampler_test.go @@ -0,0 +1,235 @@ +package stats + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/api" + "github.com/supporttools/GoClusterFS/internal/cache" + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/trim" +) + +type fakeCache struct { + calls atomic.Int64 + hits atomic.Uint64 +} + +func (f *fakeCache) Stats() cache.Stats { + f.calls.Add(1) + return cache.Stats{InodeCacheSize: 7, InodeCacheHits: f.hits.Load(), InodeCacheMisses: 1} +} + +type fakeLocks struct{ calls atomic.Int64 } + +func (f *fakeLocks) Stats() lock.Stats { + f.calls.Add(1) + return lock.Stats{Held: 2, Waiting: 1, HintCacheSize: 5, HintHits: 9, HintMisses: 1} +} + +type fakeBlocks struct { + calls atomic.Int64 + free atomic.Uint64 +} + +func (f *fakeBlocks) TotalBlocks() uint64 { return 100 } +func (f *fakeBlocks) FreeCount() uint64 { + f.calls.Add(1) + return f.free.Load() +} + +type fakeNodes struct{ calls atomic.Int64 } + +func (f *fakeNodes) LocalUUID() [16]byte { return [16]byte{0xab} } +func (f *fakeNodes) LocalSlot() int { return 0 } +func (f *fakeNodes) Slots() ([]*node.Slot, error) { + f.calls.Add(1) + return []*node.Slot{activeSlot("local", time.Now())}, nil +} + +func activeSlot(host string, beat time.Time) *node.Slot { + slot := &node.Slot{State: node.NodeStateActive, HeartbeatTime: beat.Unix(), MountTime: beat.Add(-time.Hour).Unix()} + copy(slot.Hostname[:], host) + return slot +} + +func waitFor(t *testing.T, what string, done func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !done() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestCacheResponseReportsHitRatesAsFractions(t *testing.T) { + c := cache.Stats{ + InodeCacheSize: 10, InodeCacheHits: 3, InodeCacheMisses: 1, InodeHitRate: 75, + ExtentCacheSize: 4, ExtentCacheHits: 1, ExtentCacheMisses: 1, + NegativeCacheSize: 2, + } + got := cacheResponse(c, lock.Stats{HintCacheSize: 5, HintHits: 9, HintMisses: 1}) + + want := &api.CacheStatsResponse{ + InodeCache: api.CacheHitRateInfo{Size: 10, Hits: 3, Misses: 1, HitRate: 0.75}, + ExtentCache: api.CacheHitRateInfo{Size: 4, Hits: 1, Misses: 1, HitRate: 0.5}, + NegativeCache: api.CacheHitRateInfo{Size: 2}, + LockHintCache: api.CacheHitRateInfo{Size: 5, Hits: 9, Misses: 1, HitRate: 0.9}, + } + if *got != *want { + t.Errorf("cacheResponse = %+v, want %+v", *got, *want) + } +} + +func TestLockAndJournalResponses(t *testing.T) { + l := lockResponse(lock.Stats{Held: 3, Waiting: 2, HintHits: 1, HintMisses: 3}) + if l.ActiveLocks != 3 || l.PendingLocks != 2 || l.HintCacheHitRate != 0.25 { + t.Errorf("lockResponse = %+v", *l) + } + + j := journalResponse(journal.JournalStats{State: journal.JournalStateDirty, CommitCount: 4, BytesWritten: 8192, AvailableSpace: 100}) + if j.State != "dirty" || j.CommitCount != 4 || j.BytesWritten != 8192 || j.AvailableSpace != 100 { + t.Errorf("journalResponse = %+v", *j) + } +} + +func TestTrimResponseCountsBytesAndLastDiscard(t *testing.T) { + last := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + got := trimResponse(trim.Stats{BlocksDiscarded: 3, DiscardCalls: 2, DiscardErrors: 1, PendingBlocks: 4, BatchSize: 256, LastDiscard: last}) + want := api.TrimStatsResponse{ + TrimOperations: 2, TrimBytes: 3 << 20, TrimBlocks: 3, PendingTrimBlocks: 4, + LastTrimAt: "2026-11-10T12:00:00Z", TrimErrors: 1, TrimEnabled: true, BatchSize: 256, + } + if *got != want { + t.Errorf("trimResponse = %+v, want %+v", *got, want) + } + if never := trimResponse(trim.Stats{}); never.LastTrimAt != "" { + t.Errorf("LastTrimAt before any discard = %q, want empty", never.LastTrimAt) + } +} + +func TestCapacityInfo(t *testing.T) { + got := capacityInfo(200, 50, 1<<20) + want := api.CapacityInfo{ + TotalBytes: 200 << 20, UsedBytes: 150 << 20, FreeBytes: 50 << 20, UsedPercent: 75, + TotalBlocks: 200, UsedBlocks: 150, FreeBlocks: 50, BlockSizeBytes: 1 << 20, + } + if *got != want { + t.Errorf("capacityInfo = %+v, want %+v", *got, want) + } + if empty := capacityInfo(0, 0, 1<<20); empty.UsedPercent != 0 { + t.Errorf("UsedPercent of an empty bitmap = %v, want 0", empty.UsedPercent) + } + if over := capacityInfo(10, 20, 1); over.FreeBlocks != 10 || over.UsedBlocks != 0 { + t.Errorf("free above total: %+v", *over) + } +} + +func TestClusterResponseClassifiesSlots(t *testing.T) { + now := time.Now() + stale := activeSlot("stale", now.Add(-2*node.HeartbeatTimeout)) + dead := activeSlot("dead", now.Add(-time.Minute)) + dead.State = node.NodeStateDead + peer := activeSlot("peer", now) + peer.UUID = [16]byte{0x01, 0x02} + slots := []*node.Slot{nil, activeSlot("local", now), peer, nil, stale, dead} + + got := clusterResponse([16]byte{0xab}, 1, slots, now) + + if got.NodeID != "ab000000000000000000000000000000" || got.NodeAddress != "local" || got.State != "active" || got.Role != "peer" { + t.Errorf("local node = %+v", *got) + } + if got.ClusterSize != 4 || got.HealthyNodes != 2 { + t.Errorf("ClusterSize=%d HealthyNodes=%d, want 4 and 2", got.ClusterSize, got.HealthyNodes) + } + if got.LastHeartbeatAt != now.UTC().Format(time.RFC3339) || got.JoinedAt == "" { + t.Errorf("LastHeartbeatAt=%q JoinedAt=%q", got.LastHeartbeatAt, got.JoinedAt) + } + states := map[string]string{} + for _, p := range got.Peers { + states[p.Address] = p.State + } + if want := map[string]string{"peer": "active", "stale": "stale", "dead": "dead"}; len(states) != len(want) || + states["peer"] != want["peer"] || states["stale"] != want["stale"] || states["dead"] != want["dead"] { + t.Errorf("peer states = %v, want %v", states, want) + } + if got.Peers[0].NodeID != "01020000000000000000000000000000" { + t.Errorf("peer NodeID = %q", got.Peers[0].NodeID) + } + + if alone := clusterResponse([16]byte{}, 3, make([]*node.Slot, node.MaxNodes), now); alone.State != "unknown" || alone.ClusterSize != 0 { + t.Errorf("unreadable local slot = %+v", *alone) + } +} + +func TestSamplerAnswersFromSnapshotWithoutCallingSources(t *testing.T) { + c, l, b, n := &fakeCache{}, &fakeLocks{}, &fakeBlocks{}, &fakeNodes{} + b.free.Store(40) + s := startSampler(Sources{Cache: c, Locks: l, Blocks: b, Nodes: n, BlockSize: 1 << 20}, time.Hour, time.Hour) + defer s.Stop() + + for range 100 { + s.CacheStats() + s.LockStats() + s.ClusterInfo() + s.AllStats() + s.Capacity() + } + if c.calls.Load() != 1 || l.calls.Load() != 1 || b.calls.Load() != 1 || n.calls.Load() != 1 { + t.Errorf("sources called cache=%d locks=%d free=%d slots=%d times, want 1 each (the first sample)", + c.calls.Load(), l.calls.Load(), b.calls.Load(), n.calls.Load()) + } + + if got := s.Capacity(); got == nil || got.FreeBlocks != 40 || got.UsedBlocks != 60 { + t.Errorf("Capacity = %+v", got) + } + if got := s.CacheStats(); got == nil || got.InodeCache.Size != 7 || got.LockHintCache.Size != 5 { + t.Errorf("CacheStats = %+v", got) + } + if got := s.ClusterInfo(); got == nil || got.ClusterSize != 1 { + t.Errorf("ClusterInfo = %+v", got) + } + if s.JournalStats() != nil { + t.Error("JournalStats without a journal source should be nil") + } + if got := s.TrimStats(); got == nil || got.TrimEnabled { + t.Errorf("TrimStats without a TRIM source = %+v, want a disabled report", got) + } + if s.SFBStats() != nil { + t.Error("SFBStats should be nil while SFB containers are disabled") + } + all := s.AllStats() + if all.CollectedAt == "" || all.Cache != s.CacheStats() || all.Cluster != s.ClusterInfo() || all.SFB != nil { + t.Errorf("AllStats = %+v", all) + } +} + +func TestSamplerRefreshesInBackgroundUntilStopped(t *testing.T) { + c, b, n := &fakeCache{}, &fakeBlocks{}, &fakeNodes{} + b.free.Store(10) + s := startSampler(Sources{Cache: c, Blocks: b, Nodes: n, BlockSize: 1}, 5*time.Millisecond, 10*time.Millisecond) + + c.hits.Store(42) + b.free.Store(90) + waitFor(t, "new cache hits", func() bool { return s.CacheStats().InodeCache.Hits == 42 }) + waitFor(t, "new free count", func() bool { return s.Capacity().FreeBlocks == 90 }) + if s.ClusterInfo() == nil { + t.Error("device samples dropped the cluster info") + } + + s.Stop() + s.Stop() + calls := c.calls.Load() + b.calls.Load() + n.calls.Load() + time.Sleep(50 * time.Millisecond) + if after := c.calls.Load() + b.calls.Load() + n.calls.Load(); after != calls { + t.Errorf("sources called %d more times after Stop", after-calls) + } + if s.CacheStats().InodeCache.Hits != 42 { + t.Error("the last snapshot should stay readable after Stop") + } +} diff --git a/internal/stats/stats.go b/internal/stats/stats.go deleted file mode 100644 index 922ce176..00000000 --- a/internal/stats/stats.go +++ /dev/null @@ -1,345 +0,0 @@ -package stats - -import "time" - -// Config holds configuration for the statistics collector. -type Config struct { - // CacheTTL is how long collected statistics are cached before re-collection. - // Shorter values provide fresher data but may impact performance. - // Default: 1 second - CacheTTL time.Duration - - // EnableComponentStats controls whether per-component stats are collected. - // Disable for minimal overhead in production. - // Default: true - EnableComponentStats bool -} - -// DefaultConfig returns a Config with sensible defaults. -func DefaultConfig() Config { - return Config{ - CacheTTL: 1 * time.Second, - EnableComponentStats: true, - } -} - -// HighFrequencyConfig returns a Config optimized for frequent collection. -// Use when stats need to be very fresh (e.g., real-time dashboards). -func HighFrequencyConfig() Config { - return Config{ - CacheTTL: 100 * time.Millisecond, - EnableComponentStats: true, - } -} - -// LowOverheadConfig returns a Config optimized for minimal overhead. -// Use in production when stats are infrequently accessed. -func LowOverheadConfig() Config { - return Config{ - CacheTTL: 10 * time.Second, - EnableComponentStats: false, - } -} - -// Validate checks that the configuration is valid. -func (c *Config) Validate() error { - if c.CacheTTL < 0 { - return ErrInvalidTTL - } - return nil -} - -// Snapshot contains a point-in-time snapshot of all system statistics. -// All fields are optional and may be nil if the corresponding component -// is not available or statistics collection is disabled. -type Snapshot struct { - // CollectedAt is when this snapshot was collected. - CollectedAt time.Time `json:"collected_at"` - - // Cache contains cache hit/miss statistics. - Cache *CacheStats `json:"cache,omitempty"` - - // Journal contains journal transaction statistics. - Journal *JournalStats `json:"journal,omitempty"` - - // Filesystem contains filesystem capacity and file count statistics. - Filesystem *FilesystemStats `json:"filesystem,omitempty"` - - // Lock contains lock contention statistics. - Lock *LockStats `json:"lock,omitempty"` - - // Trim contains TRIM/discard operation statistics. - Trim *TrimStats `json:"trim,omitempty"` - - // SFB contains Sparse File Bitmap (block allocation) statistics. - SFB *SFBStats `json:"sfb,omitempty"` - - // Cluster contains cluster membership and node information. - Cluster *ClusterInfo `json:"cluster,omitempty"` -} - -// CacheStats contains aggregated cache statistics. -type CacheStats struct { - // Inode cache - InodeCacheSize int `json:"inode_cache_size"` - InodeCacheHits uint64 `json:"inode_cache_hits"` - InodeCacheMisses uint64 `json:"inode_cache_misses"` - InodeHitRate float64 `json:"inode_hit_rate"` - - // Extent cache - ExtentCacheSize int `json:"extent_cache_size"` - ExtentCacheHits uint64 `json:"extent_cache_hits"` - ExtentCacheMisses uint64 `json:"extent_cache_misses"` - ExtentHitRate float64 `json:"extent_hit_rate"` - - // Negative cache (caches "not found" results) - NegativeCacheSize int `json:"negative_cache_size"` - NegativeHits uint64 `json:"negative_hits"` - NegativeMisses uint64 `json:"negative_misses"` - - // Broadcast/network invalidation stats - BroadcastEnabled bool `json:"broadcast_enabled"` - InvalidationsSent uint64 `json:"invalidations_sent,omitempty"` - InvalidationsReceived uint64 `json:"invalidations_received,omitempty"` - BroadcastDuplicates uint64 `json:"broadcast_duplicates,omitempty"` - MulticastEnabled bool `json:"multicast_enabled,omitempty"` - PeerCount int `json:"peer_count,omitempty"` -} - -// JournalStats contains journal transaction statistics. -type JournalStats struct { - // State is the current journal state (clean, dirty, recovering). - State string `json:"state"` - - // HeadOffset is the current write position in the journal. - HeadOffset uint32 `json:"head_offset"` - - // TailOffset is the oldest valid entry position. - TailOffset uint32 `json:"tail_offset"` - - // AvailableSpace is bytes available for new entries. - AvailableSpace uint32 `json:"available_space"` - - // ActiveTxnCount is the number of in-flight transactions. - ActiveTxnCount int `json:"active_txn_count"` - - // CommitCount is the total number of committed transactions. - CommitCount uint64 `json:"commit_count"` - - // BytesWritten is the total bytes written to the journal. - BytesWritten uint64 `json:"bytes_written"` - - // LastCheckpointSeq is the sequence number of the last checkpoint. - LastCheckpointSeq uint64 `json:"last_checkpoint_seq"` - - // BatchSyncCount is the number of batch syncs (for CommitBatch mode). - BatchSyncCount uint64 `json:"batch_sync_count,omitempty"` -} - -// FilesystemStats contains filesystem capacity and usage statistics. -type FilesystemStats struct { - // TotalBytes is the total filesystem capacity in bytes. - TotalBytes uint64 `json:"total_bytes"` - - // UsedBytes is the bytes currently in use. - UsedBytes uint64 `json:"used_bytes"` - - // FreeBytes is the bytes available for new data. - FreeBytes uint64 `json:"free_bytes"` - - // UsedPercent is the percentage of capacity in use. - UsedPercent float64 `json:"used_percent"` - - // TotalBlocks is the total number of blocks. - TotalBlocks uint64 `json:"total_blocks"` - - // UsedBlocks is the number of blocks in use. - UsedBlocks uint64 `json:"used_blocks"` - - // FreeBlocks is the number of free blocks. - FreeBlocks uint64 `json:"free_blocks"` - - // FileCount is the total number of files. - FileCount uint64 `json:"file_count,omitempty"` - - // DirectoryCount is the total number of directories. - DirectoryCount uint64 `json:"directory_count,omitempty"` - - // SnapshotCount is the number of active snapshots. - SnapshotCount int `json:"snapshot_count,omitempty"` -} - -// LockStats contains lock contention and hint cache statistics. -type LockStats struct { - // HintCacheHits is the number of lock hint cache hits. - HintCacheHits uint64 `json:"hint_cache_hits"` - - // HintCacheMisses is the number of lock hint cache misses. - HintCacheMisses uint64 `json:"hint_cache_misses"` - - // HintCacheHitRate is the hit rate as a ratio (0.0-1.0). - HintCacheHitRate float64 `json:"hint_cache_hit_rate"` - - // ActiveLocks is the number of currently held locks. - ActiveLocks int `json:"active_locks,omitempty"` - - // PendingLocks is the number of locks waiting to be acquired. - PendingLocks int `json:"pending_locks,omitempty"` -} - -// TrimStats contains TRIM/discard operation statistics. -type TrimStats struct { - // TrimOperations is the total number of TRIM operations performed. - TrimOperations uint64 `json:"trim_operations"` - - // TrimBytes is the total bytes trimmed. - TrimBytes uint64 `json:"trim_bytes"` - - // TrimBlocks is the total blocks trimmed. - TrimBlocks uint64 `json:"trim_blocks"` - - // PendingTrimBlocks is blocks queued for TRIM. - PendingTrimBlocks uint64 `json:"pending_trim_blocks,omitempty"` - - // LastTrimAt is when the last TRIM operation occurred. - LastTrimAt time.Time `json:"last_trim_at,omitempty"` - - // TrimErrors is the count of failed TRIM operations. - TrimErrors uint64 `json:"trim_errors,omitempty"` - - // TrimEnabled indicates if TRIM is enabled. - TrimEnabled bool `json:"trim_enabled"` - - // BatchSize is the current batch size for TRIM operations. - BatchSize int `json:"batch_size,omitempty"` -} - -// SFBStats contains Sparse File Bitmap (block allocation) statistics. -type SFBStats struct { - // TotalBlocks is the total number of blocks in the bitmap. - TotalBlocks uint64 `json:"total_blocks"` - - // AllocatedBlocks is the number of blocks currently allocated. - AllocatedBlocks uint64 `json:"allocated_blocks"` - - // FreeBlocks is the number of blocks available for allocation. - FreeBlocks uint64 `json:"free_blocks"` - - // AllocationPercent is the percentage of blocks allocated (0.0-100.0). - AllocationPercent float64 `json:"allocation_percent"` - - // FragmentationRatio estimates fragmentation (0.0-1.0, higher = more fragmented). - FragmentationRatio float64 `json:"fragmentation_ratio,omitempty"` - - // LargestFreeExtent is the size of the largest contiguous free region in blocks. - LargestFreeExtent uint64 `json:"largest_free_extent,omitempty"` - - // AllocationsTotal is the total number of allocation operations. - AllocationsTotal uint64 `json:"allocations_total"` - - // DeallocationsTotal is the total number of deallocation operations. - DeallocationsTotal uint64 `json:"deallocations_total"` - - // AllocationFailures is the count of failed allocations (e.g., no space). - AllocationFailures uint64 `json:"allocation_failures,omitempty"` -} - -// ClusterInfo contains cluster node and membership information. -type ClusterInfo struct { - // NodeID is this node's unique identifier. - NodeID string `json:"node_id"` - - // NodeAddress is this node's network address. - NodeAddress string `json:"node_address,omitempty"` - - // Role is this node's role (e.g., "primary", "replica", "standalone"). - Role string `json:"role"` - - // State is this node's current state (e.g., "active", "syncing", "offline"). - State string `json:"state"` - - // ClusterSize is the total number of nodes in the cluster. - ClusterSize int `json:"cluster_size"` - - // HealthyNodes is the number of nodes currently healthy. - HealthyNodes int `json:"healthy_nodes"` - - // LeaderID is the current cluster leader's ID (if applicable). - LeaderID string `json:"leader_id,omitempty"` - - // LastHeartbeatAt is when the last heartbeat was received/sent. - LastHeartbeatAt time.Time `json:"last_heartbeat_at,omitempty"` - - // JoinedAt is when this node joined the cluster. - JoinedAt time.Time `json:"joined_at,omitempty"` - - // Peers contains information about peer nodes. - Peers []PeerInfo `json:"peers,omitempty"` -} - -// PeerInfo contains information about a cluster peer node. -type PeerInfo struct { - // NodeID is the peer's unique identifier. - NodeID string `json:"node_id"` - - // Address is the peer's network address. - Address string `json:"address"` - - // State is the peer's current state. - State string `json:"state"` - - // LastSeen is when the peer was last seen. - LastSeen time.Time `json:"last_seen,omitempty"` - - // Latency is the round-trip latency to this peer in milliseconds. - Latency float64 `json:"latency_ms,omitempty"` -} - -// CacheStatsProvider is implemented by components that provide cache statistics. -type CacheStatsProvider interface { - // CacheStats returns current cache statistics. - // Returns nil if cache statistics are not available. - CacheStats() *CacheStats -} - -// JournalStatsProvider is implemented by components that provide journal statistics. -type JournalStatsProvider interface { - // JournalStats returns current journal statistics. - // Returns nil if journal statistics are not available. - JournalStats() *JournalStats -} - -// FilesystemStatsProvider is implemented by components that provide filesystem statistics. -type FilesystemStatsProvider interface { - // FilesystemStats returns current filesystem statistics. - // Returns nil if filesystem statistics are not available. - FilesystemStats() *FilesystemStats -} - -// LockStatsProvider is implemented by components that provide lock statistics. -type LockStatsProvider interface { - // LockStats returns current lock statistics. - // Returns nil if lock statistics are not available. - LockStats() *LockStats -} - -// TrimStatsProvider is implemented by components that provide TRIM statistics. -type TrimStatsProvider interface { - // TrimStats returns current TRIM/discard statistics. - // Returns nil if TRIM statistics are not available. - TrimStats() *TrimStats -} - -// SFBStatsProvider is implemented by components that provide block allocation statistics. -type SFBStatsProvider interface { - // SFBStats returns current Sparse File Bitmap statistics. - // Returns nil if SFB statistics are not available. - SFBStats() *SFBStats -} - -// ClusterInfoProvider is implemented by components that provide cluster information. -type ClusterInfoProvider interface { - // ClusterInfo returns current cluster membership and node information. - // Returns nil if cluster information is not available. - ClusterInfo() *ClusterInfo -} diff --git a/internal/stats/stats_test.go b/internal/stats/stats_test.go deleted file mode 100644 index c57a250f..00000000 --- a/internal/stats/stats_test.go +++ /dev/null @@ -1,615 +0,0 @@ -package stats - -import ( - "testing" - "time" -) - -// mockCacheProvider implements CacheStatsProvider for testing. -type mockCacheProvider struct { - stats *CacheStats -} - -func (m *mockCacheProvider) CacheStats() *CacheStats { - return m.stats -} - -// mockJournalProvider implements JournalStatsProvider for testing. -type mockJournalProvider struct { - stats *JournalStats -} - -func (m *mockJournalProvider) JournalStats() *JournalStats { - return m.stats -} - -// mockFilesystemProvider implements FilesystemStatsProvider for testing. -type mockFilesystemProvider struct { - stats *FilesystemStats -} - -func (m *mockFilesystemProvider) FilesystemStats() *FilesystemStats { - return m.stats -} - -// mockLockProvider implements LockStatsProvider for testing. -type mockLockProvider struct { - stats *LockStats -} - -func (m *mockLockProvider) LockStats() *LockStats { - return m.stats -} - -// mockTrimProvider implements TrimStatsProvider for testing. -type mockTrimProvider struct { - stats *TrimStats -} - -func (m *mockTrimProvider) TrimStats() *TrimStats { - return m.stats -} - -// mockSFBProvider implements SFBStatsProvider for testing. -type mockSFBProvider struct { - stats *SFBStats -} - -func (m *mockSFBProvider) SFBStats() *SFBStats { - return m.stats -} - -// mockClusterProvider implements ClusterInfoProvider for testing. -type mockClusterProvider struct { - info *ClusterInfo -} - -func (m *mockClusterProvider) ClusterInfo() *ClusterInfo { - return m.info -} - -func TestDefaultConfig(t *testing.T) { - cfg := DefaultConfig() - - if cfg.CacheTTL != 1*time.Second { - t.Errorf("expected CacheTTL 1s, got %v", cfg.CacheTTL) - } - if !cfg.EnableComponentStats { - t.Error("expected EnableComponentStats true") - } -} - -func TestConfig_Validate(t *testing.T) { - tests := []struct { - name string - config Config - wantErr error - }{ - { - name: "valid default", - config: DefaultConfig(), - wantErr: nil, - }, - { - name: "valid zero TTL", - config: Config{CacheTTL: 0}, - wantErr: nil, - }, - { - name: "invalid negative TTL", - config: Config{CacheTTL: -1 * time.Second}, - wantErr: ErrInvalidTTL, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.config.Validate() - if err != tt.wantErr { - t.Errorf("Validate() = %v, want %v", err, tt.wantErr) - } - }) - } -} - -func TestNewCollector(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - if collector == nil { - t.Fatal("NewCollector returned nil") - } -} - -func TestCollector_Collect_Empty(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - snapshot := collector.Collect() - - if snapshot == nil { - t.Fatal("Collect returned nil") - } - if snapshot.CollectedAt.IsZero() { - t.Error("CollectedAt should not be zero") - } - if snapshot.Cache != nil { - t.Error("expected Cache to be nil with no provider") - } - if snapshot.Journal != nil { - t.Error("expected Journal to be nil with no provider") - } -} - -func TestCollector_SetProviders(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - // Set cache provider - cacheStats := &CacheStats{ - InodeCacheSize: 100, - InodeHitRate: 0.95, - } - collector.SetCacheProvider(&mockCacheProvider{stats: cacheStats}) - - // Set journal provider - journalStats := &JournalStats{ - State: "clean", - CommitCount: 500, - } - collector.SetJournalProvider(&mockJournalProvider{stats: journalStats}) - - // Collect and verify - snapshot := collector.Collect() - - if snapshot.Cache == nil { - t.Fatal("expected Cache to be set") - } - if snapshot.Cache.InodeCacheSize != 100 { - t.Errorf("expected InodeCacheSize 100, got %d", snapshot.Cache.InodeCacheSize) - } - if snapshot.Cache.InodeHitRate != 0.95 { - t.Errorf("expected InodeHitRate 0.95, got %f", snapshot.Cache.InodeHitRate) - } - - if snapshot.Journal == nil { - t.Fatal("expected Journal to be set") - } - if snapshot.Journal.State != "clean" { - t.Errorf("expected State 'clean', got %q", snapshot.Journal.State) - } - if snapshot.Journal.CommitCount != 500 { - t.Errorf("expected CommitCount 500, got %d", snapshot.Journal.CommitCount) - } -} - -func TestCollector_CacheTTL(t *testing.T) { - cfg := Config{ - CacheTTL: 50 * time.Millisecond, - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - // Track how many times provider is called - callCount := 0 - provider := &countingProvider{count: &callCount} - collector.SetCacheProvider(provider) - - // First collect - snapshot1 := collector.Collect() - if callCount != 1 { - t.Errorf("expected 1 call, got %d", callCount) - } - - // Second collect within TTL - should use cache - snapshot2 := collector.Collect() - if callCount != 1 { - t.Errorf("expected still 1 call (cached), got %d", callCount) - } - - // Verify same snapshot returned (via CollectedAt) - if !snapshot1.CollectedAt.Equal(snapshot2.CollectedAt) { - t.Error("expected cached snapshot to have same CollectedAt") - } - - // Wait for TTL to expire - time.Sleep(60 * time.Millisecond) - - // Third collect after TTL - should refresh - snapshot3 := collector.Collect() - if callCount != 2 { - t.Errorf("expected 2 calls after TTL, got %d", callCount) - } - if snapshot3.CollectedAt.Equal(snapshot1.CollectedAt) { - t.Error("expected fresh snapshot after TTL") - } -} - -// countingProvider tracks how many times CacheStats is called. -type countingProvider struct { - count *int -} - -func (p *countingProvider) CacheStats() *CacheStats { - *p.count++ - return &CacheStats{InodeCacheSize: *p.count} -} - -func TestCollector_CollectFresh(t *testing.T) { - cfg := Config{ - CacheTTL: 1 * time.Hour, // Long TTL - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - callCount := 0 - provider := &countingProvider{count: &callCount} - collector.SetCacheProvider(provider) - - // First collect - collector.Collect() - if callCount != 1 { - t.Errorf("expected 1 call, got %d", callCount) - } - - // CollectFresh should bypass cache - collector.CollectFresh() - if callCount != 2 { - t.Errorf("expected 2 calls with CollectFresh, got %d", callCount) - } -} - -func TestCollector_InvalidateCache(t *testing.T) { - cfg := Config{ - CacheTTL: 1 * time.Hour, // Long TTL - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - callCount := 0 - provider := &countingProvider{count: &callCount} - collector.SetCacheProvider(provider) - - // First collect - collector.Collect() - if callCount != 1 { - t.Errorf("expected 1 call, got %d", callCount) - } - - // Invalidate cache - collector.InvalidateCache() - - // Next collect should refresh - collector.Collect() - if callCount != 2 { - t.Errorf("expected 2 calls after InvalidateCache, got %d", callCount) - } -} - -func TestCollector_DisabledComponentStats(t *testing.T) { - cfg := Config{ - CacheTTL: 1 * time.Second, - EnableComponentStats: false, // Disabled - } - collector := NewCollector(cfg) - - // Set provider - collector.SetCacheProvider(&mockCacheProvider{ - stats: &CacheStats{InodeCacheSize: 100}, - }) - - // Collect - should not include stats - snapshot := collector.Collect() - - if snapshot.Cache != nil { - t.Error("expected Cache to be nil when component stats disabled") - } -} - -func TestCollector_DirectStats(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - // No providers - should return nil - if collector.CacheStats() != nil { - t.Error("expected nil CacheStats with no provider") - } - if collector.JournalStats() != nil { - t.Error("expected nil JournalStats with no provider") - } - if collector.FilesystemStats() != nil { - t.Error("expected nil FilesystemStats with no provider") - } - if collector.LockStats() != nil { - t.Error("expected nil LockStats with no provider") - } - - // Set providers and verify direct access - collector.SetCacheProvider(&mockCacheProvider{ - stats: &CacheStats{InodeCacheSize: 42}, - }) - collector.SetLockProvider(&mockLockProvider{ - stats: &LockStats{HintCacheHits: 999}, - }) - - if stats := collector.CacheStats(); stats == nil || stats.InodeCacheSize != 42 { - t.Error("expected CacheStats with InodeCacheSize 42") - } - if stats := collector.LockStats(); stats == nil || stats.HintCacheHits != 999 { - t.Error("expected LockStats with HintCacheHits 999") - } -} - -func TestCacheInfo(t *testing.T) { - cfg := Config{ - CacheTTL: 100 * time.Millisecond, - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - // Before any collection - info := collector.CacheInfo() - if info.Valid { - t.Error("expected cache to be invalid before collection") - } - - // After collection - collector.Collect() - info = collector.CacheInfo() - if !info.Valid { - t.Error("expected cache to be valid after collection") - } - if info.Stale { - t.Error("expected cache to not be stale immediately after collection") - } - if info.TTL != 100*time.Millisecond { - t.Errorf("expected TTL 100ms, got %v", info.TTL) - } - - // Wait for TTL to expire - time.Sleep(110 * time.Millisecond) - info = collector.CacheInfo() - if !info.Stale { - t.Error("expected cache to be stale after TTL") - } -} - -func TestIsCacheValid(t *testing.T) { - cfg := Config{ - CacheTTL: 50 * time.Millisecond, - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - // Before collection - if collector.IsCacheValid() { - t.Error("expected cache invalid before collection") - } - - // After collection - collector.Collect() - if !collector.IsCacheValid() { - t.Error("expected cache valid after collection") - } - - // After TTL expires - time.Sleep(60 * time.Millisecond) - if collector.IsCacheValid() { - t.Error("expected cache invalid after TTL") - } -} - -func TestCollectWithMeta(t *testing.T) { - cfg := Config{ - CacheTTL: 100 * time.Millisecond, - EnableComponentStats: true, - } - collector := NewCollector(cfg) - - // First collection - not from cache - result1 := collector.CollectWithMeta() - if result1.FromCache { - t.Error("expected FromCache=false on first collection") - } - - // Second collection - from cache - result2 := collector.CollectWithMeta() - if !result2.FromCache { - t.Error("expected FromCache=true on second collection") - } - if result2.CacheAge < 0 { - t.Error("expected positive CacheAge") - } -} - -func TestPresetConfigs(t *testing.T) { - // Test HighFrequencyConfig - high := HighFrequencyConfig() - if high.CacheTTL != 100*time.Millisecond { - t.Errorf("HighFrequencyConfig TTL = %v, want 100ms", high.CacheTTL) - } - if err := high.Validate(); err != nil { - t.Errorf("HighFrequencyConfig validation failed: %v", err) - } - - // Test LowOverheadConfig - low := LowOverheadConfig() - if low.CacheTTL != 10*time.Second { - t.Errorf("LowOverheadConfig TTL = %v, want 10s", low.CacheTTL) - } - if low.EnableComponentStats { - t.Error("LowOverheadConfig should have EnableComponentStats=false") - } - if err := low.Validate(); err != nil { - t.Errorf("LowOverheadConfig validation failed: %v", err) - } -} - -func TestCollector_SetConfig(t *testing.T) { - collector := NewCollector(DefaultConfig()) - - // Verify initial config - if collector.Config().CacheTTL != 1*time.Second { - t.Error("expected default TTL") - } - - // Update config - newCfg := Config{CacheTTL: 5 * time.Second, EnableComponentStats: true} - collector.SetConfig(newCfg) - - if collector.Config().CacheTTL != 5*time.Second { - t.Errorf("expected new TTL 5s, got %v", collector.Config().CacheTTL) - } -} - -func TestCollector_NewProviders(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - // Set all new providers - trimStats := &TrimStats{ - TrimOperations: 100, - TrimBytes: 1024 * 1024, - TrimBlocks: 256, - TrimEnabled: true, - } - collector.SetTrimProvider(&mockTrimProvider{stats: trimStats}) - - sfbStats := &SFBStats{ - TotalBlocks: 10000, - AllocatedBlocks: 7500, - FreeBlocks: 2500, - AllocationPercent: 75.0, - } - collector.SetSFBProvider(&mockSFBProvider{stats: sfbStats}) - - clusterInfo := &ClusterInfo{ - NodeID: "node-1", - Role: "primary", - State: "active", - ClusterSize: 3, - HealthyNodes: 3, - } - collector.SetClusterProvider(&mockClusterProvider{info: clusterInfo}) - - // Collect and verify - snapshot := collector.Collect() - - // Verify TrimStats - if snapshot.Trim == nil { - t.Fatal("expected Trim to be set") - } - if snapshot.Trim.TrimOperations != 100 { - t.Errorf("expected TrimOperations 100, got %d", snapshot.Trim.TrimOperations) - } - if !snapshot.Trim.TrimEnabled { - t.Error("expected TrimEnabled true") - } - - // Verify SFBStats - if snapshot.SFB == nil { - t.Fatal("expected SFB to be set") - } - if snapshot.SFB.TotalBlocks != 10000 { - t.Errorf("expected TotalBlocks 10000, got %d", snapshot.SFB.TotalBlocks) - } - if snapshot.SFB.AllocationPercent != 75.0 { - t.Errorf("expected AllocationPercent 75.0, got %f", snapshot.SFB.AllocationPercent) - } - - // Verify ClusterInfo - if snapshot.Cluster == nil { - t.Fatal("expected Cluster to be set") - } - if snapshot.Cluster.NodeID != "node-1" { - t.Errorf("expected NodeID 'node-1', got %q", snapshot.Cluster.NodeID) - } - if snapshot.Cluster.ClusterSize != 3 { - t.Errorf("expected ClusterSize 3, got %d", snapshot.Cluster.ClusterSize) - } -} - -func TestCollector_DirectStatsNewProviders(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - // No providers - should return nil - if collector.TrimStats() != nil { - t.Error("expected nil TrimStats with no provider") - } - if collector.SFBStats() != nil { - t.Error("expected nil SFBStats with no provider") - } - if collector.ClusterInfo() != nil { - t.Error("expected nil ClusterInfo with no provider") - } - - // Set providers and verify direct access - collector.SetTrimProvider(&mockTrimProvider{ - stats: &TrimStats{TrimOperations: 42, TrimEnabled: true}, - }) - collector.SetSFBProvider(&mockSFBProvider{ - stats: &SFBStats{TotalBlocks: 999}, - }) - collector.SetClusterProvider(&mockClusterProvider{ - info: &ClusterInfo{NodeID: "test-node", State: "active"}, - }) - - if stats := collector.TrimStats(); stats == nil || stats.TrimOperations != 42 { - t.Error("expected TrimStats with TrimOperations 42") - } - if stats := collector.SFBStats(); stats == nil || stats.TotalBlocks != 999 { - t.Error("expected SFBStats with TotalBlocks 999") - } - if info := collector.ClusterInfo(); info == nil || info.NodeID != "test-node" { - t.Error("expected ClusterInfo with NodeID 'test-node'") - } -} - -func TestClusterInfo_WithPeers(t *testing.T) { - cfg := DefaultConfig() - collector := NewCollector(cfg) - - clusterInfo := &ClusterInfo{ - NodeID: "node-1", - NodeAddress: "192.168.1.10:8080", - Role: "primary", - State: "active", - ClusterSize: 3, - HealthyNodes: 3, - LeaderID: "node-1", - Peers: []PeerInfo{ - {NodeID: "node-2", Address: "192.168.1.11:8080", State: "active", Latency: 1.5}, - {NodeID: "node-3", Address: "192.168.1.12:8080", State: "active", Latency: 2.1}, - }, - } - collector.SetClusterProvider(&mockClusterProvider{info: clusterInfo}) - - snapshot := collector.Collect() - - if snapshot.Cluster == nil { - t.Fatal("expected Cluster to be set") - } - if len(snapshot.Cluster.Peers) != 2 { - t.Errorf("expected 2 peers, got %d", len(snapshot.Cluster.Peers)) - } - if snapshot.Cluster.Peers[0].NodeID != "node-2" { - t.Errorf("expected first peer NodeID 'node-2', got %q", snapshot.Cluster.Peers[0].NodeID) - } - if snapshot.Cluster.Peers[1].Latency != 2.1 { - t.Errorf("expected second peer Latency 2.1, got %f", snapshot.Cluster.Peers[1].Latency) - } -} - -func TestCollector_FilesystemProvider(t *testing.T) { - collector := NewCollector(DefaultConfig()) - collector.SetFilesystemProvider(&mockFilesystemProvider{stats: &FilesystemStats{TotalBytes: 1 << 30, UsedBytes: 1 << 20}}) - - snapshot := collector.Collect() - if snapshot.Filesystem == nil || snapshot.Filesystem.TotalBytes != 1<<30 || snapshot.Filesystem.UsedBytes != 1<<20 { - t.Fatalf("Collect().Filesystem = %+v, want the provider's stats", snapshot.Filesystem) - } - if got := collector.FilesystemStats(); got == nil || got.TotalBytes != 1<<30 { - t.Fatalf("FilesystemStats() = %+v, want the provider's stats", got) - } -} diff --git a/internal/trim/trim.go b/internal/trim/trim.go index 75e1c43f..445adc42 100644 --- a/internal/trim/trim.go +++ b/internal/trim/trim.go @@ -105,6 +105,8 @@ type Manager struct { blocksQueued uint64 blocksDiscarded uint64 discardCalls uint64 + discardErrors uint64 + lastDiscard time.Time coalesceRatio float64 // Average blocks per ioctl } @@ -265,11 +267,12 @@ func (m *Manager) flush(reason string) { func (m *Manager) discardBlocks(blocks []uint64) { ranges := coalesceBlocks(blocks) - var discarded, calls uint64 + var discarded, calls, failed uint64 for _, r := range ranges { if err := m.issueDiscard(r); err != nil { log.Printf("[trim] discard failed for range [%d, %d): %v", r.startBlock, r.startBlock+r.count, err) + failed++ continue } discarded += r.count @@ -280,6 +283,10 @@ func (m *Manager) discardBlocks(blocks []uint64) { defer m.mu.Unlock() m.blocksDiscarded += discarded m.discardCalls += calls + m.discardErrors += failed + if calls > 0 { + m.lastDiscard = time.Now() + } if len(ranges) > 0 { m.coalesceRatio = float64(discarded) / float64(len(ranges)) } @@ -374,16 +381,22 @@ func (m *Manager) Stats() Stats { BlocksQueued: m.blocksQueued, BlocksDiscarded: m.blocksDiscarded, DiscardCalls: m.discardCalls, + DiscardErrors: m.discardErrors, + LastDiscard: m.lastDiscard, PendingBlocks: uint64(len(m.pending)), CoalesceRatio: m.coalesceRatio, + BatchSize: m.config.BatchSize, } } // Stats holds TRIM manager statistics. type Stats struct { - BlocksQueued uint64 // Total blocks queued for TRIM - BlocksDiscarded uint64 // Total blocks successfully discarded - DiscardCalls uint64 // Number of BLKDISCARD ioctls issued - PendingBlocks uint64 // Blocks currently waiting - CoalesceRatio float64 // Average blocks per ioctl call + BlocksQueued uint64 // Total blocks queued for TRIM + BlocksDiscarded uint64 // Total blocks successfully discarded + DiscardCalls uint64 // Number of BLKDISCARD ioctls issued + DiscardErrors uint64 // Ranges whose BLKDISCARD failed + LastDiscard time.Time // When a BLKDISCARD last succeeded + PendingBlocks uint64 // Blocks currently waiting + CoalesceRatio float64 // Average blocks per ioctl call + BatchSize int } diff --git a/internal/trim/trim_test.go b/internal/trim/trim_test.go index fffdd481..fceff000 100644 --- a/internal/trim/trim_test.go +++ b/internal/trim/trim_test.go @@ -1,6 +1,7 @@ package trim import ( + "errors" "os" "testing" "time" @@ -347,3 +348,30 @@ func TestDuplicateBlockDiscardedOnce(t *testing.T) { t.Errorf("discards = %v, want %v", calls, want) } } + +func TestStatsCountFailedDiscardsAndLastSuccess(t *testing.T) { + dev, cleanup := createTestDevice(t) + defer cleanup() + + m := NewManager(dev, Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + if offset == 10*BlockSize { + return errors.New("device refused") + } + return nil + }}) + m.Start() + m.Queue(5) + m.Queue(10) + before := time.Now() + if err := m.Stop(); err != nil { + t.Fatal(err) + } + + got := m.Stats() + if got.DiscardCalls != 1 || got.BlocksDiscarded != 1 || got.DiscardErrors != 1 || got.BatchSize != 1000 { + t.Errorf("Stats = %+v, want 1 call, 1 block, 1 error, batch 1000", got) + } + if got.LastDiscard.Before(before) { + t.Errorf("LastDiscard = %v, want after %v", got.LastDiscard, before) + } +} diff --git a/tests/integration/api_stats_test.go b/tests/integration/api_stats_test.go new file mode 100644 index 00000000..15ef52c7 --- /dev/null +++ b/tests/integration/api_stats_test.go @@ -0,0 +1,165 @@ +package integration + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/api" +) + +var apiClient = &http.Client{Timeout: 2 * time.Second} + +func getAPI(t *testing.T, url string, into any) int { + t.Helper() + resp, err := apiClient.Get(url) + if err != nil { + return 0 + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK && into != nil { + if err := json.NewDecoder(resp.Body).Decode(into); err != nil { + t.Fatalf("GET %s: %v", url, err) + } + } + return resp.StatusCode +} + +func waitForAPI(t *testing.T, base string) { + t.Helper() + if _, ok := waitFor(10*time.Second, func() bool { return getAPI(t, base+"/health", nil) == http.StatusOK }); !ok { + t.Fatalf("API at %s never answered", base) + } +} + +func metricValue(body, name string) (float64, bool) { + for _, line := range strings.Split(body, "\n") { + if v, ok := strings.CutPrefix(line, name+" "); ok { + f, err := strconv.ParseFloat(v, 64) + return f, err == nil + } + } + return 0, false +} + +func TestAPIStatsReportLiveValues(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + addrs := []string{freeLoopbackAddr(t), freeLoopbackAddr(t)} + for addrs[1] == addrs[0] { + addrs[1] = freeLoopbackAddr(t) + } + for i, addr := range addrs { + cluster.Node(i).ExtraArgs = []string{"-api-addr", addr, "-broadcast=false", "-trim-age", "1s"} + } + first, second := "http://"+addrs[0], "http://"+addrs[1] + + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("Failed to format and mount node 0: %v", err) + } + waitForAPI(t, first) + + fs := cluster.Node(0) + data := bytes.Repeat([]byte("stats"), 3<<20/5) + if err := fs.WriteFile("kept.bin", data); err != nil { + t.Fatalf("Write failed: %v", err) + } + if got, err := fs.ReadFile("kept.bin"); err != nil || !bytes.Equal(got, data) { + t.Fatalf("Read back failed: %v", err) + } + if err := fs.WriteFile("freed.bin", data); err != nil { + t.Fatalf("Write failed: %v", err) + } + if err := fs.Remove("freed.bin"); err != nil { + t.Fatalf("Remove failed: %v", err) + } + if err := fs.Mkdir("dir"); err != nil { + t.Fatalf("Mkdir failed: %v", err) + } + if _, err := fs.Stat("kept.bin"); err != nil { + t.Fatalf("Stat failed: %v", err) + } + + var ( + cacheStats api.CacheStatsResponse + journalStats api.JournalStatsResponse + lockStats api.LockStatsResponse + trimStats api.TrimStatsResponse + ) + _, ok := waitFor(15*time.Second, func() bool { + return getAPI(t, first+"/api/v1/stats/cache", &cacheStats) == http.StatusOK && + getAPI(t, first+"/api/v1/stats/journal", &journalStats) == http.StatusOK && + getAPI(t, first+"/api/v1/stats/lock", &lockStats) == http.StatusOK && + getAPI(t, first+"/api/v1/stats/trim", &trimStats) == http.StatusOK && + cacheStats.InodeCache.Hits+cacheStats.InodeCache.Misses > 0 && + journalStats.CommitCount > 0 && journalStats.BytesWritten > 0 && + lockStats.HintCacheHits+lockStats.HintCacheMisses > 0 && + trimStats.TrimBlocks > 0 + }) + if !ok { + t.Fatalf("stats never showed node 0's I/O:\ncache=%+v\njournal=%+v\nlock=%+v\ntrim=%+v", cacheStats, journalStats, lockStats, trimStats) + } + if journalStats.State != "clean" && journalStats.State != "dirty" { + t.Errorf("journal state = %q, want clean or dirty", journalStats.State) + } + if !trimStats.TrimEnabled || trimStats.TrimBytes != trimStats.TrimBlocks<<20 { + t.Errorf("trim = %+v", trimStats) + } + + if code := getAPI(t, first+"/api/v1/stats/sfb", nil); code != http.StatusNotFound { + t.Errorf("GET /api/v1/stats/sfb = %d, want 404 while SFB is disabled", code) + } + var all api.AllStatsResponse + if code := getAPI(t, first+"/api/v1/stats/all", &all); code != http.StatusOK || all.Cache == nil || all.Journal == nil || all.Cluster == nil || all.SFB != nil { + t.Errorf("GET /api/v1/stats/all = %d %+v", code, all) + } + var local api.ClusterInfoResponse + if code := getAPI(t, first+"/api/v1/stats/cluster", &local); code != http.StatusOK || local.NodeID == "" || local.State != "active" || local.ClusterSize != 1 { + t.Fatalf("node 0 cluster = %d %+v", code, local) + } + + resp, err := apiClient.Get(first + "/metrics") + if err != nil { + t.Fatalf("GET /metrics: %v", err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + metrics := string(body) + for _, name := range []string{"goclusterfs_journal_commits_total", "goclusterfs_journal_bytes_written_total", "goclusterfs_cache_inode_size", "goclusterfs_trim_blocks_total", "goclusterfs_fs_size_bytes", "goclusterfs_fs_free_bytes", "goclusterfs_cluster_size", "goclusterfs_trim_enabled"} { + if v, ok := metricValue(metrics, name); !ok || v <= 0 { + t.Errorf("/metrics %s = %v (found %v), want > 0", name, v, ok) + } + } + if v, ok := metricValue(metrics, "goclusterfs_journal_state"); !ok || (v != 0 && v != 1) { + t.Errorf("/metrics goclusterfs_journal_state = %v (found %v), want 0 or 1", v, ok) + } + if strings.Contains(metrics, "goclusterfs_sfb_") { + t.Error("/metrics exports sfb metrics while SFB is disabled") + } + + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("Failed to mount node 1: %v", err) + } + waitForAPI(t, second) + + var view api.ClusterInfoResponse + if code := getAPI(t, second+"/api/v1/stats/cluster", &view); code != http.StatusOK { + t.Fatalf("node 1 GET /api/v1/stats/cluster = %d", code) + } + if view.ClusterSize != 2 || view.HealthyNodes != 2 || view.NodeID == local.NodeID || + len(view.Peers) != 1 || view.Peers[0].NodeID != local.NodeID || view.Peers[0].State != "active" { + t.Errorf("node 1 cluster = %+v, want 2 healthy nodes with node 0 (%s) as an active peer", view, local.NodeID) + } + + var status api.StatusResponse + if code := getAPI(t, second+"/api/v1/status", &status); code != http.StatusOK || status.Capacity == nil { + t.Fatalf("node 1 GET /api/v1/status = %d %+v", code, status) + } + if c := status.Capacity; c.TotalBytes == 0 || c.UsedBlocks < 3 || c.UsedBlocks+c.FreeBlocks != c.TotalBlocks { + t.Errorf("node 1 capacity = %+v, want node 0's 3 MB file counted", *c) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 87e59770..6b26b955 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2327,3 +2327,80 @@ scenarios: priority: go_only environments: go: api_test.go::TestMountRejectsBadAPISettingsBeforeOpeningDevice + + - id: api-stats-live-values + name: API stats report live values + description: After writes, reads, a delete and a mkdir, /api/v1/stats/{cache,journal,lock,trim,all,cluster} and /metrics show non-zero counters, /stats/sfb answers 404, and a second node sees both nodes and the first node's blocks + priority: go_only + environments: + go: api_stats_test.go::TestAPIStatsReportLiveValues + + - id: stats-sampler-serves-snapshot + name: Stats requests never call the managers + description: Repeated StatsProvider and Capacity calls are served from the last snapshot; sources are read only by the sampler + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestSamplerAnswersFromSnapshotWithoutCallingSources + + - id: stats-sampler-refresh + name: Stats sampler refreshes until stopped + description: Counter and device samples pick up new values in the background; Stop is idempotent and no source is read after it + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestSamplerRefreshesInBackgroundUntilStopped + + - id: stats-cache-hit-rates + name: Cache stats report hit rates as fractions + description: Inode, extent, negative and lock hint cache stats carry sizes and counts, with hit rates from 0 to 1 + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestCacheResponseReportsHitRatesAsFractions + + - id: stats-lock-journal + name: Lock and journal stats + description: Lock stats carry held and waiting counts; journal stats carry a lowercase state and the commit counters + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestLockAndJournalResponses + + - id: stats-trim + name: TRIM stats + description: TRIM stats convert discarded blocks to bytes and report the last discard time, empty before any discard + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestTrimResponseCountsBytesAndLastDiscard + + - id: stats-capacity + name: Capacity from the block bitmap + description: Capacity derives used blocks and bytes from total and free blocks, with 0% for an empty bitmap + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestCapacityInfo + + - id: stats-cluster-slots + name: Cluster stats classify registry slots + description: Cluster stats count non-empty slots, report the local node and list peers as active, stale or dead + priority: go_only + environments: + go: internal/stats/sampler_test.go::TestClusterResponseClassifiesSlots + + - id: lock-stats-held-waiting + name: Lock manager counts held and waiting locks + description: Stats reports the holder's lock and a blocked Acquire on another node until the lock is released + priority: go_only + environments: + go: internal/lock/safety_test.go::TestStatsCountsHeldAndWaitingLocks + + - id: trim-stats-errors + name: TRIM counts failed discards + description: A failed BLKDISCARD range counts as an error; a successful one sets the last discard time + priority: go_only + environments: + go: internal/trim/trim_test.go::TestStatsCountFailedDiscardsAndLastSuccess + + - id: api-metrics-capacity + name: /metrics exports capacity + description: With a status provider, /metrics exports goclusterfs_fs_size_bytes and goclusterfs_fs_free_bytes + priority: go_only + environments: + go: internal/api/prometheus_test.go::TestMetricsEndpoint_ExportsCapacity