diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a00a208..8b1b0fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - CI gates: unit tests with the race detector and coverage, golangci-lint (`make lint`), govulncheck (`make vuln`), and a `go mod tidy` check. - Release workflow: pushing a `v*` tag publishes a GitHub Release with `linux/amd64` and `linux/arm64` archives, checksums, and the matching `CHANGELOG.md` section. `scripts/dist.sh` builds the same archives locally. - `goclusterfs-fsck -version` prints the release version, commit, and commit date. -- HTTP API in `goclusterfs-mount` (`-api`, on by default): `/health`, `/metrics`, `/api/v1/status` and `/api/v1/stats/...`. It listens on `127.0.0.1:8080` (`-api-addr`). Mount refuses a non-loopback `-api-addr` or `-api-cors '*'` without `-api-token`, and checks the API flags before it opens the device. +- HTTP API in `goclusterfs-mount` (`-api`, on by default): `/health`, `/metrics`, `/api/v1/status` and `/api/v1/stats/...`. It listens on `127.0.0.1:8080` (`-api-addr`). Mount refuses a non-loopback `-api-addr` or `-api-cors '*'` without `-api-token`, and checks the API flags before it opens the device. `/metrics` uses the Prometheus client library and adds Go runtime and process metrics. `-api-rate-limit` limits requests per client IP (off by default). - `goclusterfs-mount -version`, which prints the same version string as `goclusterfs-fsck -version`. `/api/v1/status` reports it too. - Mount logs a `WARNING` line at startup when `-broadcast` is on without `-broadcast-secret`. - Design, deployment, operations, and troubleshooting docs in `docs/`. diff --git a/README.md b/README.md index f68d1aae..3a75df2b 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ With `-broadcast` on and no `-broadcast-secret`, mount logs a `WARNING` line at | `-api-addr` | `127.0.0.1:8080` | API listen address | | `-api-token` | (empty) | Bearer token required on `/api/v1/...` | | `-api-cors` | (empty) | Comma-separated CORS allowed origins | +| `-api-rate-limit` | `0` (off) | Requests per second per client IP, with bursts of twice that | The API only listens on this host by default. Mount refuses to start when `-api-addr` can be reached from other hosts and `-api-token` is empty. See [HTTP API Flags](docs/API-CONFIG.md#http-api-flags). diff --git a/cmd/mount/main.go b/cmd/mount/main.go index db9a6849..d16507f0 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "log" + "math" "os" "os/signal" "strings" @@ -78,17 +79,25 @@ func (p *fsStatusProvider) Capacity() *api.CapacityInfo { } } -func apiConfig(addr, token, corsList string) api.Config { +func apiConfig(addr, token, corsList string, rateLimit float64) api.Config { var origins []string for _, origin := range strings.Split(corsList, ",") { if origin = strings.TrimSpace(origin); origin != "" { origins = append(origins, origin) } } - return api.DefaultConfig(). + cfg := api.DefaultConfig(). WithListenAddr(addr). WithAuthToken(token). WithCORSOrigins(origins...) + if rateLimit != 0 { + burst := 1 + if rateLimit > 0 && rateLimit <= api.MaxRateLimit { + burst = int(math.Ceil(2 * rateLimit)) + } + cfg = cfg.WithRateLimit(rateLimit, burst) + } + return cfg } func main() { @@ -133,6 +142,7 @@ func main() { apiAddr = flag.String("api-addr", "127.0.0.1:8080", "API server listen address; any non-loopback address needs -api-token") apiToken = flag.String("api-token", "", "Bearer token for API authentication (empty=disabled, loopback only)") apiCORS = flag.String("api-cors", "", "Comma-separated CORS allowed origins (empty=disabled)") + apiRate = flag.Float64("api-rate-limit", 0, "API requests per second per client IP, with bursts of twice that (0=off)") ) flag.Parse() @@ -147,7 +157,7 @@ func main() { os.Exit(1) } - apiCfg := apiConfig(*apiAddr, *apiToken, *apiCORS) + apiCfg := apiConfig(*apiAddr, *apiToken, *apiCORS, *apiRate) if *apiEnabled { if err := apiCfg.Validate(); err != nil { log.Fatalf("Invalid API settings: %v", err) @@ -457,6 +467,7 @@ func main() { apiServer.RegisterHealthChecker(api.DefaultHealthChecker()) if err := apiServer.Start(ctx); err != nil { log.Printf("Warning: Failed to start API server: %v", err) + apiServer.Stop() apiServer = nil } else { log.Printf("API server listening on %s (auth=%v, cors=%v)", diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 370f1c83..2248d3ba 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -90,13 +90,16 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For | `-api-addr` | string | `127.0.0.1:8080` | Listen address | | `-api-token` | string | (empty) | Bearer token for `/api/v1/...` requests | | `-api-cors` | string | (empty) | Comma-separated CORS allowed origins, or `*` | +| `-api-rate-limit` | float | `0` (off) | Requests per second per client IP, with bursts of twice that, up to 1,000,000. Over the limit, the API answers 429 | -The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. Mount does not feed statistics to the API yet, so `/api/v1/stats/...` answers 503, `/health` only checks the API itself, and `/metrics` only reports `goclusterfs_up`. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer `. `/health` and `/metrics` never need the token. +The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status` and `GET /api/v1/stats/...`. Mount does not feed statistics to the API yet, so `/api/v1/stats/...` answers 503, `/health` only checks the API itself, and `/metrics` only reports `goclusterfs_up` plus Go runtime and process metrics. With `-api-token` set, `/api/v1/...` requests need `Authorization: Bearer `. `/health` and `/metrics` never need the token. Without `-api-token`, the API answers only requests whose `Host` header is `localhost`, a `127.x` address or `[::1]`, so a web page cannot reach it through DNS rebinding, and mount refuses `-api-cors '*'`. List the dashboard's origin instead, for example `-api-cors http://localhost:5173`. By default the API only accepts connections from this host. To serve other hosts, set `-api-addr` (for example `:8080` or `10.0.0.5:8080`) and `-api-token`. Mount exits before it opens the device when the address is not loopback (`127.0.0.0/8`, `::1` or `localhost`) and the token is empty. The token is a command-line flag, so local users can read it with `ps`. +The rate limit counts requests per connecting IP address, and per /64 for IPv6. It ignores `X-Forwarded-For`, so behind a proxy every client shares the proxy's limit. `/health` and `/metrics` are never limited. The limiter tracks at most 65,536 clients at once; while it is full, new clients get 429 until idle clients' buckets refill and are dropped, about 2 seconds (longer below 0.5 requests per second). + Each mount on a host needs its own `-api-addr`. When the port is taken, mount logs a warning and runs without the API. --- diff --git a/go.mod b/go.mod index 27b26549..2d556f6f 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,19 @@ go 1.26.0 require github.com/hanwen/go-fuse/v2 v2.9.0 -require golang.org/x/sys v0.28.0 +require ( + github.com/prometheus/client_golang v1.23.2 + golang.org/x/sys v0.35.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/go.sum b/go.sum index 5185160b..63cc61ba 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,50 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58= github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/config.go b/internal/api/config.go index 2b0b570f..bd8322a1 100644 --- a/internal/api/config.go +++ b/internal/api/config.go @@ -9,6 +9,9 @@ import ( "time" ) +// MaxRateLimit is the highest per-client rate limit, in requests per second. +const MaxRateLimit = 1e6 + // Config holds the configuration for the API server. type Config struct { // ListenAddr is the address to listen on (e.g., "127.0.0.1:8080" or ":8080"). @@ -41,18 +44,33 @@ type Config struct { // ShutdownTimeout is the maximum duration to wait for graceful shutdown. // Default: 10s ShutdownTimeout time.Duration + + // RateLimitEnabled enables rate limiting middleware. + // Default: false + RateLimitEnabled bool + + // RateLimitRequestsPerSecond is the sustained rate limit per client. + // Default: 100 + RateLimitRequestsPerSecond float64 + + // RateLimitBurst is the maximum burst size (token bucket capacity). + // Default: 200 + RateLimitBurst int } // DefaultConfig returns a Config with sensible defaults. func DefaultConfig() Config { return Config{ - ListenAddr: "127.0.0.1:8080", - AuthToken: "", - CORSOrigins: nil, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 60 * time.Second, - ShutdownTimeout: 10 * time.Second, + ListenAddr: "127.0.0.1:8080", + AuthToken: "", + CORSOrigins: nil, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + ShutdownTimeout: 10 * time.Second, + RateLimitEnabled: false, + RateLimitRequestsPerSecond: 100, + RateLimitBurst: 200, } } @@ -118,6 +136,16 @@ func (c *Config) Validate() error { } } + // Validate rate limit configuration + if c.RateLimitEnabled { + if !(c.RateLimitRequestsPerSecond > 0) || c.RateLimitRequestsPerSecond > MaxRateLimit { + return fmt.Errorf("%w: requests per second must be more than 0 and at most %g", ErrInvalidRateLimit, MaxRateLimit) + } + if c.RateLimitBurst <= 0 { + return fmt.Errorf("%w: burst must be > 0", ErrInvalidRateLimit) + } + } + return nil } @@ -165,3 +193,11 @@ func (c Config) WithCORSOrigins(origins ...string) Config { c.CORSOrigins = origins return c } + +// WithRateLimit returns a copy of the config with rate limiting enabled. +func (c Config) WithRateLimit(requestsPerSecond float64, burst int) Config { + c.RateLimitEnabled = true + c.RateLimitRequestsPerSecond = requestsPerSecond + c.RateLimitBurst = burst + return c +} diff --git a/internal/api/errors.go b/internal/api/errors.go index 20d7c6f3..e1588104 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -27,4 +27,7 @@ var ( // ErrShutdownTimeout is returned when graceful shutdown times out. ErrShutdownTimeout = errors.New("api: shutdown timed out") + + // ErrInvalidRateLimit is returned when rate limit configuration is invalid. + ErrInvalidRateLimit = errors.New("api: invalid rate limit configuration") ) diff --git a/internal/api/middleware.go b/internal/api/middleware.go index ce904213..5fdafbb5 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -3,9 +3,13 @@ package api import ( "crypto/subtle" "log" + "math" "net" "net/http" + "slices" + "strconv" "strings" + "sync" "time" ) @@ -143,6 +147,163 @@ func CORSMiddleware(origins []string) Middleware { } } +// MaxRateLimitClients bounds the limiter's memory. While this many clients +// have live buckets, requests from new clients get 429. +const MaxRateLimitClients = 65536 + +// RateLimiter implements a token bucket rate limiter per client. +type RateLimiter struct { + mu sync.Mutex + clients map[string]*tokenBucket + rate float64 // tokens per second + burst int // max tokens (bucket capacity) + maxClients int + idle time.Duration // a bucket idle this long is full again, so it is dropped + stop chan struct{} + stopped chan struct{} + stopOnce sync.Once +} + +// tokenBucket tracks rate limit state for a single client. +type tokenBucket struct { + tokens float64 + lastSeen time.Time +} + +// NewRateLimiter creates a new rate limiter with the specified rate and burst. +func NewRateLimiter(requestsPerSecond float64, burst int) *RateLimiter { + idle := time.Hour + if refill := float64(burst) / requestsPerSecond; refill < idle.Seconds() { + idle = time.Duration(refill * float64(time.Second)) + } + rl := &RateLimiter{ + clients: make(map[string]*tokenBucket), + rate: requestsPerSecond, + burst: burst, + maxClients: MaxRateLimitClients, + idle: idle, + stop: make(chan struct{}), + stopped: make(chan struct{}), + } + go rl.cleanup(min(max(idle, time.Second), time.Minute)) + return rl +} + +// Allow checks if a request from the given client should be allowed. +func (rl *RateLimiter) Allow(key string) bool { + rl.mu.Lock() + defer rl.mu.Unlock() + + now := time.Now() + bucket, exists := rl.clients[key] + + if !exists { + if len(rl.clients) >= rl.maxClients { + return false + } + // New client starts with a full bucket + rl.clients[key] = &tokenBucket{ + tokens: float64(rl.burst) - 1, // Consume one token for this request + lastSeen: now, + } + return true + } + + // Calculate tokens to add based on time elapsed + elapsed := now.Sub(bucket.lastSeen).Seconds() + bucket.tokens += elapsed * rl.rate + if bucket.tokens > float64(rl.burst) { + bucket.tokens = float64(rl.burst) + } + bucket.lastSeen = now + + // Check if we have tokens available + if bucket.tokens < 1 { + return false + } + + bucket.tokens-- + return true +} + +func (rl *RateLimiter) cleanup(every time.Duration) { + defer close(rl.stopped) + ticker := time.NewTicker(every) + defer ticker.Stop() + + for { + select { + case <-rl.stop: + return + case now := <-ticker.C: + rl.sweep(now) + } + } +} + +func (rl *RateLimiter) sweep(now time.Time) { + rl.mu.Lock() + defer rl.mu.Unlock() + for key, bucket := range rl.clients { + if now.Sub(bucket.lastSeen) >= rl.idle { + delete(rl.clients, key) + } + } +} + +// Stop ends the cleanup goroutine. It is safe to call more than once. +func (rl *RateLimiter) Stop() { + rl.stopOnce.Do(func() { close(rl.stop) }) +} + +func (rl *RateLimiter) retryAfter() string { + return strconv.Itoa(int(min(max(math.Ceil(1/rl.rate), 1), 3600))) +} + +// RateLimitMiddleware returns a middleware that enforces rate limiting per client. +// If limiter is nil, no rate limiting is applied. Paths in skipPaths are never limited. +func RateLimitMiddleware(limiter *RateLimiter, skipPaths ...string) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if limiter == nil || slices.Contains(skipPaths, r.URL.Path) { + next.ServeHTTP(w, r) + return + } + + if !limiter.Allow(rateLimitKey(r)) { + w.Header().Set("Retry-After", limiter.retryAfter()) + writeError(w, http.StatusTooManyRequests, "rate limit exceeded", "please slow down") + return + } + + next.ServeHTTP(w, r) + }) + } +} + +// rateLimitKey groups IPv6 clients by /64, the block one host usually controls. +func rateLimitKey(r *http.Request) string { + host := getClientIP(r) + ip := net.ParseIP(host) + switch { + case ip == nil: + return host + case ip.To4() != nil: + return ip.String() + default: + return ip.Mask(net.CIDRMask(64, 128)).String() + "/64" + } +} + +// getClientIP returns the connection's IP. Proxy headers are ignored: any client can set them. +func getClientIP(r *http.Request) string { + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return ip +} + // LoopbackHostMiddleware rejects requests whose Host header is not a loopback // name, so a DNS-rebinding web page cannot read an API that has no token. func LoopbackHostMiddleware(next http.Handler) http.Handler { diff --git a/internal/api/middleware_test.go b/internal/api/middleware_test.go index 992fca55..a152d267 100644 --- a/internal/api/middleware_test.go +++ b/internal/api/middleware_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "math" "net/http" "testing" "time" @@ -322,3 +323,264 @@ func TestCORSMiddleware_Preflight(t *testing.T) { t.Error("expected Access-Control-Allow-Methods header") } } + +// Rate Limiting Tests + +func TestRateLimiter_Allow(t *testing.T) { + // Create a rate limiter with 10 req/s and burst of 5 + rl := NewRateLimiter(10, 5) + + // First 5 requests should be allowed (burst) + for i := 0; i < 5; i++ { + if !rl.Allow("192.168.1.1") { + t.Errorf("request %d should be allowed", i+1) + } + } + + // 6th request should be denied (burst exhausted) + if rl.Allow("192.168.1.1") { + t.Error("6th request should be denied") + } + + // Different IP should be allowed (separate bucket) + if !rl.Allow("192.168.1.2") { + t.Error("request from different IP should be allowed") + } +} + +func TestRateLimiter_TokenRefill(t *testing.T) { + // Create a rate limiter with 1000 req/s (fast refill for testing) + rl := NewRateLimiter(1000, 2) + + // Exhaust the bucket + rl.Allow("192.168.1.1") + rl.Allow("192.168.1.1") + if rl.Allow("192.168.1.1") { + t.Error("should be rate limited after exhausting burst") + } + + // Wait for refill (at 1000/s, 1ms = 1 token) + time.Sleep(2 * time.Millisecond) + + // Should be allowed again after refill + if !rl.Allow("192.168.1.1") { + t.Error("should be allowed after token refill") + } +} + +func TestRateLimitMiddleware_Disabled(t *testing.T) { + // No rate limiting when not configured + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + // Make many requests - all should succeed + for i := 0; i < 50; i++ { + resp, err := http.Get("http://" + server.Addr() + "/health") + if err != nil { + t.Fatalf("request error: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("request %d: expected 200, got %d", i, resp.StatusCode) + } + } +} + +func TestRateLimitMiddleware_Enabled(t *testing.T) { + // Enable rate limiting with small burst for easy testing + cfg := DefaultConfig().WithListenAddr(":0").WithRateLimit(1, 3) + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + // First 3 requests should succeed (burst) + for i := 0; i < 3; i++ { + resp, err := http.Get("http://" + server.Addr() + "/api/v1/status") + if err != nil { + t.Fatalf("request error: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("request %d: expected 200, got %d", i+1, resp.StatusCode) + } + } + + // 4th request should be rate limited + resp, err := http.Get("http://" + server.Addr() + "/api/v1/status") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusTooManyRequests { + t.Errorf("4th request: expected 429, got %d", resp.StatusCode) + } + + // Check Retry-After header + if retry := resp.Header.Get("Retry-After"); retry != "1" { + t.Errorf("expected Retry-After header '1', got %q", retry) + } +} + +func TestGetClientIP_RemoteAddr(t *testing.T) { + req, _ := http.NewRequest("GET", "/", nil) + req.RemoteAddr = "192.168.1.100:12345" + + ip := getClientIP(req) + if ip != "192.168.1.100" { + t.Errorf("expected '192.168.1.100', got %q", ip) + } +} + +func TestGetClientIP_IgnoresProxyHeaders(t *testing.T) { + req, _ := http.NewRequest("GET", "/", nil) + req.RemoteAddr = "10.0.0.1:12345" + req.Header.Set("X-Forwarded-For", "203.0.113.50") + req.Header.Set("X-Real-IP", "198.51.100.78") + + if ip := getClientIP(req); ip != "10.0.0.1" { + t.Errorf("expected the connection's IP 10.0.0.1, got %q", ip) + } +} + +func TestRateLimitIgnoresForwardedFor(t *testing.T) { + srv := startServer(t, DefaultConfig().WithListenAddr("127.0.0.1:0").WithRateLimit(0.001, 1)) + if got := get(t, srv, "/api/v1/status", "", map[string]string{"X-Forwarded-For": "203.0.113.1"}).StatusCode; got != http.StatusOK { + t.Fatalf("first request: status %d, want 200", got) + } + if got := get(t, srv, "/api/v1/status", "", map[string]string{"X-Forwarded-For": "203.0.113.2"}).StatusCode; got != http.StatusTooManyRequests { + t.Fatalf("second request with a new X-Forwarded-For: status %d, want 429", got) + } +} + +func TestRateLimiterStopEndsCleanup(t *testing.T) { + rl := NewRateLimiter(1, 1) + rl.Stop() + rl.Stop() + select { + case <-rl.stopped: + case <-time.After(time.Second): + t.Fatal("cleanup goroutine still running after Stop") + } +} + +func TestServerStopStopsRateLimiter(t *testing.T) { + for _, start := range []bool{true, false} { + srv, err := NewServer(DefaultConfig().WithListenAddr("127.0.0.1:0").WithRateLimit(10, 20)) + if err != nil { + t.Fatal(err) + } + if start { + if err := srv.Start(context.Background()); err != nil { + t.Fatal(err) + } + } + srv.Stop() + select { + case <-srv.rateLimiter.stopped: + case <-time.After(time.Second): + t.Fatalf("started=%v: rate limiter cleanup still running after Stop", start) + } + } +} + +func TestRateLimiterCapsClients(t *testing.T) { + rl := NewRateLimiter(1, 2) + defer rl.Stop() + rl.maxClients = 2 + if !rl.Allow("a") || !rl.Allow("b") { + t.Fatal("first two clients refused") + } + if rl.Allow("c") { + t.Fatal("third client allowed past the cap") + } + if !rl.Allow("a") { + t.Fatal("known client refused at the cap") + } +} + +func TestRateLimiterSweepDropsFullBuckets(t *testing.T) { + rl := NewRateLimiter(1, 2) + defer rl.Stop() + rl.Allow("a") + count := func() int { + rl.mu.Lock() + defer rl.mu.Unlock() + return len(rl.clients) + } + rl.sweep(time.Now().Add(time.Second)) + if count() != 1 { + t.Fatal("bucket dropped before it refilled") + } + rl.sweep(time.Now().Add(3 * time.Second)) + if count() != 0 { + t.Fatal("full bucket not dropped") + } +} + +func TestRateLimitKey(t *testing.T) { + key := func(remote string) string { + req, _ := http.NewRequest("GET", "/", nil) + req.RemoteAddr = remote + return rateLimitKey(req) + } + if key("[2001:db8::1]:1") != key("[2001:db8::2]:2") { + t.Error("addresses in one IPv6 /64 got different keys") + } + if key("[2001:db8::1]:1") == key("[2001:db8:0:1::1]:1") { + t.Error("different IPv6 /64s share a key") + } + if got := key("[::ffff:192.0.2.1]:1"); got != "192.0.2.1" { + t.Errorf("mapped IPv4 key = %q, want 192.0.2.1", got) + } + if got := key("192.0.2.1:1"); got != "192.0.2.1" { + t.Errorf("IPv4 key = %q, want 192.0.2.1", got) + } +} + +func TestRateLimitSkipsHealthAndMetrics(t *testing.T) { + srv := startServer(t, DefaultConfig().WithListenAddr("127.0.0.1:0").WithRateLimit(0.25, 1)) + for i := 0; i < 3; i++ { + for _, path := range []string{"/health", "/metrics"} { + if got := get(t, srv, path, "", nil).StatusCode; got != http.StatusOK { + t.Fatalf("%s request %d: status %d, want 200", path, i+1, got) + } + } + } + get(t, srv, "/api/v1/status", "", nil) + resp := get(t, srv, "/api/v1/status", "", nil) + if resp.StatusCode != http.StatusTooManyRequests || resp.Header.Get("Retry-After") != "4" { + t.Fatalf("second status request: %d with Retry-After %q, want 429 with 4", resp.StatusCode, resp.Header.Get("Retry-After")) + } +} + +func TestConfigRejectsBadRateLimits(t *testing.T) { + for _, rate := range []float64{0, -1, math.NaN(), math.Inf(1), 2 * MaxRateLimit} { + cfg := DefaultConfig().WithRateLimit(rate, 1) + if err := cfg.Validate(); err == nil { + t.Errorf("rate %v accepted", rate) + } + } + cfg := DefaultConfig().WithRateLimit(MaxRateLimit, 1) + if err := cfg.Validate(); err != nil { + t.Errorf("rate %v refused: %v", MaxRateLimit, err) + } +} diff --git a/internal/api/prometheus.go b/internal/api/prometheus.go new file mode 100644 index 00000000..c22a8786 --- /dev/null +++ b/internal/api/prometheus.go @@ -0,0 +1,568 @@ +package api + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + namespace = "goclusterfs" +) + +// MetricsHandler returns an HTTP handler for Prometheus metrics scraping. +// It uses a custom collector that pulls fresh stats from the StatsProvider. +func (s *Server) MetricsHandler() http.Handler { + registry := prometheus.NewRegistry() + + // Register the filesystem collector + collector := newFSCollector(s) + registry.MustRegister(collector) + + // Also register standard process and Go metrics if desired + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + registry.MustRegister(collectors.NewGoCollector()) + + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{ + EnableOpenMetrics: true, + MaxRequestsInFlight: 2, + }) +} + +// fsCollector implements prometheus.Collector interface to expose filesystem stats. +type fsCollector struct { + server *Server + + // Descriptors for all metrics + upDesc *prometheus.Desc + + // Cache metrics + cacheInodeSizeDesc *prometheus.Desc + cacheInodeHitsDesc *prometheus.Desc + cacheInodeMissesDesc *prometheus.Desc + cacheInodeHitRateDesc *prometheus.Desc + cacheExtentSizeDesc *prometheus.Desc + cacheExtentHitsDesc *prometheus.Desc + cacheExtentMissesDesc *prometheus.Desc + cacheExtentHitRateDesc *prometheus.Desc + cacheNegativeSizeDesc *prometheus.Desc + cacheNegativeHitsDesc *prometheus.Desc + cacheNegativeMissesDesc *prometheus.Desc + cacheNegativeHitRateDesc *prometheus.Desc + cacheLockHintSizeDesc *prometheus.Desc + cacheLockHintHitsDesc *prometheus.Desc + cacheLockHintMissesDesc *prometheus.Desc + cacheLockHintHitRateDesc *prometheus.Desc + + // Journal metrics + journalStateDesc *prometheus.Desc + journalHeadOffsetDesc *prometheus.Desc + journalTailOffsetDesc *prometheus.Desc + journalAvailableSpaceDesc *prometheus.Desc + journalActiveTxnCountDesc *prometheus.Desc + journalCommitCountDesc *prometheus.Desc + journalBytesWrittenDesc *prometheus.Desc + journalLastCheckpointDesc *prometheus.Desc + journalBatchSyncCountDesc *prometheus.Desc + + // Trim metrics + trimOperationsDesc *prometheus.Desc + trimBytesDesc *prometheus.Desc + trimBlocksDesc *prometheus.Desc + trimPendingBlocksDesc *prometheus.Desc + trimErrorsDesc *prometheus.Desc + trimEnabledDesc *prometheus.Desc + + // Lock metrics + lockHintCacheHitsDesc *prometheus.Desc + lockHintCacheMissesDesc *prometheus.Desc + lockHintCacheHitRateDesc *prometheus.Desc + lockActiveLocksDesc *prometheus.Desc + lockPendingLocksDesc *prometheus.Desc + + // SFB (Sparse File Bitmap) metrics + sfbTotalBlocksDesc *prometheus.Desc + sfbAllocatedBlocksDesc *prometheus.Desc + sfbFreeBlocksDesc *prometheus.Desc + sfbAllocationPercentDesc *prometheus.Desc + sfbFragmentationDesc *prometheus.Desc + sfbLargestFreeExtentDesc *prometheus.Desc + sfbAllocationsDesc *prometheus.Desc + sfbDeallocationsDesc *prometheus.Desc + sfbAllocationFailuresDesc *prometheus.Desc + + // Cluster metrics + clusterSizeDesc *prometheus.Desc + clusterHealthyNodesDesc *prometheus.Desc + clusterIsLeaderDesc *prometheus.Desc +} + +func newFSCollector(s *Server) *fsCollector { + return &fsCollector{ + server: s, + + // Up metric + upDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "up"), + "Whether GoClusterFS is up and running", + nil, nil, + ), + + // Cache - Inode + cacheInodeSizeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "inode_size"), + "Number of entries in the inode cache", + nil, nil, + ), + cacheInodeHitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "inode_hits_total"), + "Total inode cache hits", + nil, nil, + ), + cacheInodeMissesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "inode_misses_total"), + "Total inode cache misses", + nil, nil, + ), + cacheInodeHitRateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "inode_hit_rate"), + "Inode cache hit rate (0.0-1.0)", + nil, nil, + ), + + // Cache - Extent + cacheExtentSizeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "extent_size"), + "Number of entries in the extent cache", + nil, nil, + ), + cacheExtentHitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "extent_hits_total"), + "Total extent cache hits", + nil, nil, + ), + cacheExtentMissesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "extent_misses_total"), + "Total extent cache misses", + nil, nil, + ), + cacheExtentHitRateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "extent_hit_rate"), + "Extent cache hit rate (0.0-1.0)", + nil, nil, + ), + + // Cache - Negative + cacheNegativeSizeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "negative_size"), + "Number of entries in the negative cache", + nil, nil, + ), + cacheNegativeHitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "negative_hits_total"), + "Total negative cache hits", + nil, nil, + ), + cacheNegativeMissesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "negative_misses_total"), + "Total negative cache misses", + nil, nil, + ), + cacheNegativeHitRateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "negative_hit_rate"), + "Negative cache hit rate (0.0-1.0)", + nil, nil, + ), + + // Cache - Lock Hint + cacheLockHintSizeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "lock_hint_size"), + "Number of entries in the lock hint cache", + nil, nil, + ), + cacheLockHintHitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "lock_hint_hits_total"), + "Total lock hint cache hits", + nil, nil, + ), + cacheLockHintMissesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "lock_hint_misses_total"), + "Total lock hint cache misses", + nil, nil, + ), + cacheLockHintHitRateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cache", "lock_hint_hit_rate"), + "Lock hint cache hit rate (0.0-1.0)", + nil, nil, + ), + + // Journal + journalStateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "state"), + "Journal state (1=open, 0=closed)", + nil, nil, + ), + journalHeadOffsetDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "head_offset"), + "Journal head offset in bytes", + nil, nil, + ), + journalTailOffsetDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "tail_offset"), + "Journal tail offset in bytes", + nil, nil, + ), + journalAvailableSpaceDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "available_space_bytes"), + "Available space in journal in bytes", + nil, nil, + ), + journalActiveTxnCountDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "active_transactions"), + "Number of active transactions", + nil, nil, + ), + journalCommitCountDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "commits_total"), + "Total number of commits", + nil, nil, + ), + journalBytesWrittenDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "bytes_written_total"), + "Total bytes written to journal", + nil, nil, + ), + journalLastCheckpointDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "last_checkpoint_seq"), + "Last checkpoint sequence number", + nil, nil, + ), + journalBatchSyncCountDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "journal", "batch_syncs_total"), + "Total number of batch syncs", + nil, nil, + ), + + // Trim + trimOperationsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "operations_total"), + "Total TRIM operations performed", + nil, nil, + ), + trimBytesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "bytes_total"), + "Total bytes trimmed", + nil, nil, + ), + trimBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "blocks_total"), + "Total blocks trimmed", + nil, nil, + ), + trimPendingBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "pending_blocks"), + "Number of blocks pending TRIM", + nil, nil, + ), + trimErrorsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "errors_total"), + "Total TRIM errors", + nil, nil, + ), + trimEnabledDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "trim", "enabled"), + "Whether TRIM is enabled (1) or disabled (0)", + nil, nil, + ), + + // Lock + lockHintCacheHitsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "lock", "hint_cache_hits_total"), + "Total lock hint cache hits", + nil, nil, + ), + lockHintCacheMissesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "lock", "hint_cache_misses_total"), + "Total lock hint cache misses", + nil, nil, + ), + lockHintCacheHitRateDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "lock", "hint_cache_hit_rate"), + "Lock hint cache hit rate (0.0-1.0)", + nil, nil, + ), + lockActiveLocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "lock", "active_locks"), + "Number of currently active locks", + nil, nil, + ), + lockPendingLocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "lock", "pending_locks"), + "Number of pending lock requests", + nil, nil, + ), + + // SFB (Sparse File Bitmap) + sfbTotalBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "total_blocks"), + "Total number of blocks in the filesystem", + nil, nil, + ), + sfbAllocatedBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "allocated_blocks"), + "Number of allocated blocks", + nil, nil, + ), + sfbFreeBlocksDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "free_blocks"), + "Number of free blocks", + nil, nil, + ), + sfbAllocationPercentDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "allocation_ratio"), + "Block allocation ratio (0.0-1.0)", + nil, nil, + ), + sfbFragmentationDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "fragmentation_ratio"), + "Block fragmentation ratio (0.0-1.0)", + nil, nil, + ), + sfbLargestFreeExtentDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "largest_free_extent_blocks"), + "Largest contiguous free extent in blocks", + nil, nil, + ), + sfbAllocationsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "allocations_total"), + "Total block allocations", + nil, nil, + ), + sfbDeallocationsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "deallocations_total"), + "Total block deallocations", + nil, nil, + ), + sfbAllocationFailuresDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sfb", "allocation_failures_total"), + "Total block allocation failures", + nil, nil, + ), + + // Cluster + clusterSizeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cluster", "size"), + "Total number of nodes in the cluster", + nil, nil, + ), + clusterHealthyNodesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cluster", "healthy_nodes"), + "Number of healthy nodes in the cluster", + nil, nil, + ), + clusterIsLeaderDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "cluster", "is_leader"), + "Whether this node is the cluster leader (1) or not (0)", + nil, nil, + ), + } +} + +// Describe sends the super-set of all possible descriptors of metrics +// collected by this Collector to the provided channel. +func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.upDesc + + // Cache + ch <- c.cacheInodeSizeDesc + ch <- c.cacheInodeHitsDesc + ch <- c.cacheInodeMissesDesc + ch <- c.cacheInodeHitRateDesc + ch <- c.cacheExtentSizeDesc + ch <- c.cacheExtentHitsDesc + ch <- c.cacheExtentMissesDesc + ch <- c.cacheExtentHitRateDesc + ch <- c.cacheNegativeSizeDesc + ch <- c.cacheNegativeHitsDesc + ch <- c.cacheNegativeMissesDesc + ch <- c.cacheNegativeHitRateDesc + ch <- c.cacheLockHintSizeDesc + ch <- c.cacheLockHintHitsDesc + ch <- c.cacheLockHintMissesDesc + ch <- c.cacheLockHintHitRateDesc + + // Journal + ch <- c.journalStateDesc + ch <- c.journalHeadOffsetDesc + ch <- c.journalTailOffsetDesc + ch <- c.journalAvailableSpaceDesc + ch <- c.journalActiveTxnCountDesc + ch <- c.journalCommitCountDesc + ch <- c.journalBytesWrittenDesc + ch <- c.journalLastCheckpointDesc + ch <- c.journalBatchSyncCountDesc + + // Trim + ch <- c.trimOperationsDesc + ch <- c.trimBytesDesc + ch <- c.trimBlocksDesc + ch <- c.trimPendingBlocksDesc + ch <- c.trimErrorsDesc + ch <- c.trimEnabledDesc + + // Lock + ch <- c.lockHintCacheHitsDesc + ch <- c.lockHintCacheMissesDesc + ch <- c.lockHintCacheHitRateDesc + ch <- c.lockActiveLocksDesc + ch <- c.lockPendingLocksDesc + + // SFB + ch <- c.sfbTotalBlocksDesc + ch <- c.sfbAllocatedBlocksDesc + ch <- c.sfbFreeBlocksDesc + ch <- c.sfbAllocationPercentDesc + ch <- c.sfbFragmentationDesc + ch <- c.sfbLargestFreeExtentDesc + ch <- c.sfbAllocationsDesc + ch <- c.sfbDeallocationsDesc + ch <- c.sfbAllocationFailuresDesc + + // Cluster + ch <- c.clusterSizeDesc + ch <- c.clusterHealthyNodesDesc + ch <- c.clusterIsLeaderDesc +} + +// Collect is called by the Prometheus registry when collecting metrics. +// It fetches fresh stats from the server's StatsProvider. +func (c *fsCollector) Collect(ch chan<- prometheus.Metric) { + // Always report up metric + ch <- prometheus.MustNewConstMetric(c.upDesc, prometheus.GaugeValue, 1) + + // Get stats provider + provider := c.server.GetStatsProvider() + if provider == nil { + return + } + + // Collect cache stats + if cache := provider.CacheStats(); cache != nil { + c.collectCacheStats(ch, cache) + } + + // Collect journal stats + if journal := provider.JournalStats(); journal != nil { + c.collectJournalStats(ch, journal) + } + + // Collect trim stats + if trim := provider.TrimStats(); trim != nil { + c.collectTrimStats(ch, trim) + } + + // Collect lock stats + if lock := provider.LockStats(); lock != nil { + c.collectLockStats(ch, lock) + } + + // Collect SFB stats + if sfb := provider.SFBStats(); sfb != nil { + c.collectSFBStats(ch, sfb) + } + + // Collect cluster stats + if cluster := provider.ClusterInfo(); cluster != nil { + c.collectClusterStats(ch, cluster) + } +} + +func (c *fsCollector) collectCacheStats(ch chan<- prometheus.Metric, stats *CacheStatsResponse) { + // Inode cache + ch <- prometheus.MustNewConstMetric(c.cacheInodeSizeDesc, prometheus.GaugeValue, float64(stats.InodeCache.Size)) + ch <- prometheus.MustNewConstMetric(c.cacheInodeHitsDesc, prometheus.CounterValue, float64(stats.InodeCache.Hits)) + ch <- prometheus.MustNewConstMetric(c.cacheInodeMissesDesc, prometheus.CounterValue, float64(stats.InodeCache.Misses)) + ch <- prometheus.MustNewConstMetric(c.cacheInodeHitRateDesc, prometheus.GaugeValue, stats.InodeCache.HitRate) + + // Extent cache + ch <- prometheus.MustNewConstMetric(c.cacheExtentSizeDesc, prometheus.GaugeValue, float64(stats.ExtentCache.Size)) + ch <- prometheus.MustNewConstMetric(c.cacheExtentHitsDesc, prometheus.CounterValue, float64(stats.ExtentCache.Hits)) + ch <- prometheus.MustNewConstMetric(c.cacheExtentMissesDesc, prometheus.CounterValue, float64(stats.ExtentCache.Misses)) + ch <- prometheus.MustNewConstMetric(c.cacheExtentHitRateDesc, prometheus.GaugeValue, stats.ExtentCache.HitRate) + + // Negative cache + ch <- prometheus.MustNewConstMetric(c.cacheNegativeSizeDesc, prometheus.GaugeValue, float64(stats.NegativeCache.Size)) + ch <- prometheus.MustNewConstMetric(c.cacheNegativeHitsDesc, prometheus.CounterValue, float64(stats.NegativeCache.Hits)) + ch <- prometheus.MustNewConstMetric(c.cacheNegativeMissesDesc, prometheus.CounterValue, float64(stats.NegativeCache.Misses)) + ch <- prometheus.MustNewConstMetric(c.cacheNegativeHitRateDesc, prometheus.GaugeValue, stats.NegativeCache.HitRate) + + // Lock hint cache + ch <- prometheus.MustNewConstMetric(c.cacheLockHintSizeDesc, prometheus.GaugeValue, float64(stats.LockHintCache.Size)) + ch <- prometheus.MustNewConstMetric(c.cacheLockHintHitsDesc, prometheus.CounterValue, float64(stats.LockHintCache.Hits)) + ch <- prometheus.MustNewConstMetric(c.cacheLockHintMissesDesc, prometheus.CounterValue, float64(stats.LockHintCache.Misses)) + ch <- prometheus.MustNewConstMetric(c.cacheLockHintHitRateDesc, prometheus.GaugeValue, stats.LockHintCache.HitRate) +} + +func (c *fsCollector) collectJournalStats(ch chan<- prometheus.Metric, stats *JournalStatsResponse) { + // Convert state to numeric (1=open, 0=other) + stateValue := 0.0 + if stats.State == "open" { + stateValue = 1.0 + } + ch <- prometheus.MustNewConstMetric(c.journalStateDesc, prometheus.GaugeValue, stateValue) + ch <- prometheus.MustNewConstMetric(c.journalHeadOffsetDesc, prometheus.GaugeValue, float64(stats.HeadOffset)) + ch <- prometheus.MustNewConstMetric(c.journalTailOffsetDesc, prometheus.GaugeValue, float64(stats.TailOffset)) + ch <- prometheus.MustNewConstMetric(c.journalAvailableSpaceDesc, prometheus.GaugeValue, float64(stats.AvailableSpace)) + ch <- prometheus.MustNewConstMetric(c.journalActiveTxnCountDesc, prometheus.GaugeValue, float64(stats.ActiveTxnCount)) + ch <- prometheus.MustNewConstMetric(c.journalCommitCountDesc, prometheus.CounterValue, float64(stats.CommitCount)) + ch <- prometheus.MustNewConstMetric(c.journalBytesWrittenDesc, prometheus.CounterValue, float64(stats.BytesWritten)) + ch <- prometheus.MustNewConstMetric(c.journalLastCheckpointDesc, prometheus.GaugeValue, float64(stats.LastCheckpointSeq)) + ch <- prometheus.MustNewConstMetric(c.journalBatchSyncCountDesc, prometheus.CounterValue, float64(stats.BatchSyncCount)) +} + +func (c *fsCollector) collectTrimStats(ch chan<- prometheus.Metric, stats *TrimStatsResponse) { + ch <- prometheus.MustNewConstMetric(c.trimOperationsDesc, prometheus.CounterValue, float64(stats.TrimOperations)) + ch <- prometheus.MustNewConstMetric(c.trimBytesDesc, prometheus.CounterValue, float64(stats.TrimBytes)) + ch <- prometheus.MustNewConstMetric(c.trimBlocksDesc, prometheus.CounterValue, float64(stats.TrimBlocks)) + ch <- prometheus.MustNewConstMetric(c.trimPendingBlocksDesc, prometheus.GaugeValue, float64(stats.PendingTrimBlocks)) + ch <- prometheus.MustNewConstMetric(c.trimErrorsDesc, prometheus.CounterValue, float64(stats.TrimErrors)) + + enabledValue := 0.0 + if stats.TrimEnabled { + enabledValue = 1.0 + } + ch <- prometheus.MustNewConstMetric(c.trimEnabledDesc, prometheus.GaugeValue, enabledValue) +} + +func (c *fsCollector) collectLockStats(ch chan<- prometheus.Metric, stats *LockStatsResponse) { + ch <- prometheus.MustNewConstMetric(c.lockHintCacheHitsDesc, prometheus.CounterValue, float64(stats.HintCacheHits)) + ch <- prometheus.MustNewConstMetric(c.lockHintCacheMissesDesc, prometheus.CounterValue, float64(stats.HintCacheMisses)) + ch <- prometheus.MustNewConstMetric(c.lockHintCacheHitRateDesc, prometheus.GaugeValue, stats.HintCacheHitRate) + ch <- prometheus.MustNewConstMetric(c.lockActiveLocksDesc, prometheus.GaugeValue, float64(stats.ActiveLocks)) + ch <- prometheus.MustNewConstMetric(c.lockPendingLocksDesc, prometheus.GaugeValue, float64(stats.PendingLocks)) +} + +func (c *fsCollector) collectSFBStats(ch chan<- prometheus.Metric, stats *SFBStatsResponse) { + ch <- prometheus.MustNewConstMetric(c.sfbTotalBlocksDesc, prometheus.GaugeValue, float64(stats.TotalBlocks)) + ch <- prometheus.MustNewConstMetric(c.sfbAllocatedBlocksDesc, prometheus.GaugeValue, float64(stats.AllocatedBlocks)) + ch <- prometheus.MustNewConstMetric(c.sfbFreeBlocksDesc, prometheus.GaugeValue, float64(stats.FreeBlocks)) + ch <- prometheus.MustNewConstMetric(c.sfbAllocationPercentDesc, prometheus.GaugeValue, stats.AllocationPercent/100.0) + ch <- prometheus.MustNewConstMetric(c.sfbFragmentationDesc, prometheus.GaugeValue, stats.FragmentationRatio) + ch <- prometheus.MustNewConstMetric(c.sfbLargestFreeExtentDesc, prometheus.GaugeValue, float64(stats.LargestFreeExtent)) + ch <- prometheus.MustNewConstMetric(c.sfbAllocationsDesc, prometheus.CounterValue, float64(stats.AllocationsTotal)) + ch <- prometheus.MustNewConstMetric(c.sfbDeallocationsDesc, prometheus.CounterValue, float64(stats.DeallocationsTotal)) + ch <- prometheus.MustNewConstMetric(c.sfbAllocationFailuresDesc, prometheus.CounterValue, float64(stats.AllocationFailures)) +} + +func (c *fsCollector) collectClusterStats(ch chan<- prometheus.Metric, stats *ClusterInfoResponse) { + ch <- prometheus.MustNewConstMetric(c.clusterSizeDesc, prometheus.GaugeValue, float64(stats.ClusterSize)) + ch <- prometheus.MustNewConstMetric(c.clusterHealthyNodesDesc, prometheus.GaugeValue, float64(stats.HealthyNodes)) + + isLeader := 0.0 + if stats.Role == "leader" { + isLeader = 1.0 + } + ch <- prometheus.MustNewConstMetric(c.clusterIsLeaderDesc, prometheus.GaugeValue, isLeader) +} diff --git a/internal/api/prometheus_test.go b/internal/api/prometheus_test.go new file mode 100644 index 00000000..01503cc5 --- /dev/null +++ b/internal/api/prometheus_test.go @@ -0,0 +1,447 @@ +package api + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestMetricsEndpoint_BasicUp(t *testing.T) { + server := startTestServer(t) + defer server.Stop() + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + + content := string(body) + + // Should always have the up metric + if !strings.Contains(content, "goclusterfs_up 1") { + t.Error("expected goclusterfs_up metric") + } + + // Should have Go collector metrics + if !strings.Contains(content, "go_goroutines") { + t.Error("expected go_goroutines metric from Go collector") + } + + // Should have process metrics + if !strings.Contains(content, "process_") { + t.Error("expected process_* metrics from process collector") + } +} + +func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // Set up mock provider with all stats + server.SetStatsProvider(&mockStatsProvider{ + cache: &CacheStatsResponse{ + InodeCache: CacheHitRateInfo{ + Size: 1000, + Hits: 800, + Misses: 200, + HitRate: 0.8, + }, + ExtentCache: CacheHitRateInfo{ + Size: 500, + Hits: 400, + Misses: 100, + HitRate: 0.8, + }, + NegativeCache: CacheHitRateInfo{ + Size: 100, + Hits: 50, + Misses: 50, + HitRate: 0.5, + }, + LockHintCache: CacheHitRateInfo{ + Size: 200, + Hits: 180, + Misses: 20, + HitRate: 0.9, + }, + }, + journal: &JournalStatsResponse{ + State: "open", + HeadOffset: 4096, + TailOffset: 2048, + AvailableSpace: 1048576, + ActiveTxnCount: 5, + CommitCount: 1000, + BytesWritten: 10485760, + LastCheckpointSeq: 999, + BatchSyncCount: 100, + }, + trim: &TrimStatsResponse{ + TrimOperations: 500, + TrimBytes: 52428800, + TrimBlocks: 50, + PendingTrimBlocks: 5, + TrimErrors: 2, + TrimEnabled: true, + }, + lock: &LockStatsResponse{ + HintCacheHits: 900, + HintCacheMisses: 100, + HintCacheHitRate: 0.9, + ActiveLocks: 10, + PendingLocks: 2, + }, + sfb: &SFBStatsResponse{ + TotalBlocks: 1000, + AllocatedBlocks: 600, + FreeBlocks: 400, + AllocationPercent: 60.0, + FragmentationRatio: 0.1, + LargestFreeExtent: 100, + AllocationsTotal: 5000, + DeallocationsTotal: 4400, + AllocationFailures: 5, + }, + cluster: &ClusterInfoResponse{ + NodeID: "node-1", + Role: "leader", + State: "active", + ClusterSize: 3, + HealthyNodes: 3, + }, + }) + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + + content := string(body) + + // Cache metrics + expectedMetrics := []string{ + "goclusterfs_cache_inode_size 1000", + "goclusterfs_cache_inode_hits_total 800", + "goclusterfs_cache_inode_misses_total 200", + "goclusterfs_cache_inode_hit_rate 0.8", + "goclusterfs_cache_extent_size 500", + "goclusterfs_cache_extent_hits_total 400", + "goclusterfs_cache_negative_size 100", + "goclusterfs_cache_lock_hint_size 200", + "goclusterfs_cache_lock_hint_hit_rate 0.9", + + // Journal metrics + "goclusterfs_journal_state 1", + "goclusterfs_journal_head_offset 4096", + "goclusterfs_journal_tail_offset 2048", + "goclusterfs_journal_available_space_bytes 1.048576e+06", + "goclusterfs_journal_active_transactions 5", + "goclusterfs_journal_commits_total 1000", + "goclusterfs_journal_bytes_written_total 1.048576e+07", + "goclusterfs_journal_batch_syncs_total 100", + + // Trim metrics + "goclusterfs_trim_operations_total 500", + "goclusterfs_trim_bytes_total 5.24288e+07", + "goclusterfs_trim_blocks_total 50", + "goclusterfs_trim_pending_blocks 5", + "goclusterfs_trim_errors_total 2", + "goclusterfs_trim_enabled 1", + + // Lock metrics + "goclusterfs_lock_hint_cache_hits_total 900", + "goclusterfs_lock_hint_cache_misses_total 100", + "goclusterfs_lock_hint_cache_hit_rate 0.9", + "goclusterfs_lock_active_locks 10", + "goclusterfs_lock_pending_locks 2", + + // SFB metrics + "goclusterfs_sfb_total_blocks 1000", + "goclusterfs_sfb_allocated_blocks 600", + "goclusterfs_sfb_free_blocks 400", + "goclusterfs_sfb_allocations_total 5000", + "goclusterfs_sfb_deallocations_total 4400", + "goclusterfs_sfb_allocation_failures_total 5", + + // Cluster metrics + "goclusterfs_cluster_size 3", + "goclusterfs_cluster_healthy_nodes 3", + "goclusterfs_cluster_is_leader 1", + } + + for _, metric := range expectedMetrics { + if !strings.Contains(content, metric) { + t.Errorf("expected metric %q not found in output", metric) + } + } +} + +func TestMetricsEndpoint_JournalClosedState(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // Set up mock provider with closed journal + server.SetStatsProvider(&mockStatsProvider{ + journal: &JournalStatsResponse{ + State: "closed", + }, + }) + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + + content := string(body) + + // Journal state should be 0 for closed + if !strings.Contains(content, "goclusterfs_journal_state 0") { + t.Error("expected journal_state to be 0 for closed state") + } +} + +func TestMetricsEndpoint_TrimDisabled(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // Set up mock provider with TRIM disabled + server.SetStatsProvider(&mockStatsProvider{ + trim: &TrimStatsResponse{ + TrimEnabled: false, + }, + }) + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + + content := string(body) + + // Trim enabled should be 0 + if !strings.Contains(content, "goclusterfs_trim_enabled 0") { + t.Error("expected trim_enabled to be 0") + } +} + +func TestMetricsEndpoint_ClusterFollower(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // Set up mock provider with follower node + server.SetStatsProvider(&mockStatsProvider{ + cluster: &ClusterInfoResponse{ + NodeID: "node-2", + Role: "follower", + State: "active", + ClusterSize: 3, + HealthyNodes: 3, + }, + }) + + if err := server.Start(context.Background()); err != nil { + t.Fatalf("Start error: %v", err) + } + defer server.Stop() + time.Sleep(10 * time.Millisecond) + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body error: %v", err) + } + + content := string(body) + + // is_leader should be 0 for follower + if !strings.Contains(content, "goclusterfs_cluster_is_leader 0") { + t.Error("expected is_leader to be 0 for follower") + } +} + +func TestMetricsEndpoint_ContentType(t *testing.T) { + server := startTestServer(t) + defer server.Stop() + + resp, err := http.Get("http://" + server.Addr() + "/metrics") + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + contentType := resp.Header.Get("Content-Type") + // Prometheus handler returns text/plain with version info + if !strings.HasPrefix(contentType, "text/plain") && + !strings.HasPrefix(contentType, "application/openmetrics-text") { + t.Errorf("unexpected content-type: %q", contentType) + } +} + +func TestMetricsEndpoint_MethodNotAllowed(t *testing.T) { + server := startTestServer(t) + defer server.Stop() + + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, "http://"+server.Addr()+"/metrics", nil) + if err != nil { + t.Fatalf("create request error: %v", err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("request error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("expected status 405, got %d", resp.StatusCode) + } +} + +func TestMetricsHandler_LazyInit(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // Get handler multiple times - should not panic and return non-nil + handler1 := server.GetMetricsHandler() + if handler1 == nil { + t.Error("expected non-nil handler on first call") + } + + handler2 := server.GetMetricsHandler() + if handler2 == nil { + t.Error("expected non-nil handler on second call") + } + + // Both calls should succeed without panic +} + +func TestFSCollector_Describe(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + collector := newFSCollector(server) + ch := make(chan *prometheus.Desc, 100) + + go func() { + collector.Describe(ch) + close(ch) + }() + + count := 0 + for range ch { + count++ + } + + // Should have descriptors for all metrics + // up (1) + cache (16) + journal (9) + trim (6) + lock (5) + sfb (9) + cluster (3) = 49 + expectedMinimum := 40 // Allow some flexibility + if count < expectedMinimum { + t.Errorf("expected at least %d descriptors, got %d", expectedMinimum, count) + } +} + +func TestFSCollector_Collect_NoProvider(t *testing.T) { + cfg := DefaultConfig().WithListenAddr(":0") + server, err := NewServer(cfg) + if err != nil { + t.Fatalf("NewServer error: %v", err) + } + + // No stats provider set + collector := newFSCollector(server) + ch := make(chan prometheus.Metric, 100) + + go func() { + collector.Collect(ch) + close(ch) + }() + + count := 0 + for range ch { + count++ + } + + // Should only have the up metric when no provider + if count != 1 { + t.Errorf("expected 1 metric (up), got %d", count) + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 690e1b4e..5d510894 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -270,15 +270,13 @@ func (s *Server) handleStatsAll(w http.ResponseWriter, r *http.Request) { } // handleMetrics handles GET /metrics requests for Prometheus scraping. +// Delegates to the Prometheus metrics handler for proper metric exposition. func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { writeError(w, http.StatusMethodNotAllowed, "method not allowed", "use GET") return } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusOK) - w.Write([]byte("# HELP goclusterfs_up Indicates if GoClusterFS is up\n")) - w.Write([]byte("# TYPE goclusterfs_up gauge\n")) - w.Write([]byte("goclusterfs_up 1\n")) + // Delegate to the Prometheus handler + s.GetMetricsHandler().ServeHTTP(w, r) } diff --git a/internal/api/server.go b/internal/api/server.go index 8d13356c..db5176fe 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -29,6 +29,12 @@ type Server struct { // Stats provider for statistics collection (optional) statsProvider StatsProvider + // Rate limiter (optional) + rateLimiter *RateLimiter + + // Prometheus metrics handler (lazily initialized) + metricsHandler http.Handler + // Lifecycle state started bool startTime time.Time @@ -48,11 +54,17 @@ func NewServer(cfg Config) (*Server, error) { router: http.NewServeMux(), } + // Create rate limiter if enabled + if cfg.RateLimitEnabled { + s.rateLimiter = NewRateLimiter(cfg.RateLimitRequestsPerSecond, cfg.RateLimitBurst) + } + // Register routes s.registerRoutes() - // Build middleware chain (order: logging -> loopback Host check -> CORS -> auth) - middlewares := []Middleware{LoggingMiddleware} + // Build middleware chain (order: rate limit -> logging -> loopback Host check -> CORS -> auth). + // Probes and scrapers on loopback share one bucket, so /health and /metrics skip the limiter. + middlewares := []Middleware{RateLimitMiddleware(s.rateLimiter, "/health", "/metrics"), LoggingMiddleware} if cfg.AuthToken == "" && cfg.LoopbackOnly() { middlewares = append(middlewares, LoopbackHostMiddleware) } @@ -111,6 +123,9 @@ func (s *Server) Start(ctx context.Context) error { // ShutdownTimeout before forcefully closing them. func (s *Server) Stop() error { s.stopOnce.Do(func() { + if s.rateLimiter != nil { + s.rateLimiter.Stop() + } s.mu.Lock() if !s.started { s.mu.Unlock() @@ -256,3 +271,14 @@ func (s *Server) GetStatsProvider() StatsProvider { defer s.mu.RUnlock() return s.statsProvider } + +// GetMetricsHandler returns the Prometheus metrics HTTP handler. +// The handler is lazily initialized on first call. +func (s *Server) GetMetricsHandler() http.Handler { + s.mu.Lock() + defer s.mu.Unlock() + if s.metricsHandler == nil { + s.metricsHandler = s.MetricsHandler() + } + return s.metricsHandler +} diff --git a/tests/integration/api_test.go b/tests/integration/api_test.go index 7b7eb3c8..69e7aff6 100644 --- a/tests/integration/api_test.go +++ b/tests/integration/api_test.go @@ -144,6 +144,10 @@ func TestMountRejectsBadAPISettingsBeforeOpeningDevice(t *testing.T) { {"-api-addr=127.0.0.1", "Invalid API settings"}, {"-api-cors=not-a-url", "Invalid API settings"}, {"-api-cors=*", "-api-cors"}, + {"-api-rate-limit=-1", "Invalid API settings"}, + {"-api-rate-limit=NaN", "Invalid API settings"}, + {"-api-rate-limit=+Inf", "Invalid API settings"}, + {"-api-rate-limit=1e19", "Invalid API settings"}, } { out, err := runMount(t, "-device", missing, "-mount", dir, tt.flag) if err == nil || !strings.Contains(out, tt.want) || strings.Contains(out, "Failed to open device") { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index b95b270d..17070f5f 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2285,7 +2285,7 @@ scenarios: - id: api-settings-checked-before-device name: Mount rejects bad API settings early - description: An -api-addr without a port, an invalid -api-cors, or -api-cors * without a token stop mount before it opens the device + description: An -api-addr without a port, an invalid -api-cors, -api-cors * without a token, or an -api-rate-limit that is negative, NaN, infinite or too large stop mount before it opens the device priority: go_only environments: go: api_test.go::TestMountRejectsBadAPISettingsBeforeOpeningDevice