Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
- A shutdown step that takes longer than 2 seconds logs its name and duration, and a shutdown still running after 8 seconds logs every goroutine's stack.

- `-cache-inodes`, `-cache-extents`, `-cache-negative` and `-broadcast-max-peers` set the cache and broadcast table sizes, and `/metrics` reports each limit next to its size.
- A node that hears no broadcasts from a node the registry shows mounted, for 3 `-broadcast-heartbeat` intervals (90 seconds by default), logs a warning naming it, reports `broadcast-peers` as degraded on `/health`, and counts it in `goclusterfs_broadcast_silent_peers`. A network that silently drops multicast was invisible before.

### Changed

Expand Down
10 changes: 10 additions & 0 deletions cmd/goclusterfs-mount/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"time"

"github.com/supporttools/GoClusterFS/internal/api"
)
Expand All @@ -14,6 +15,7 @@ type healthSources struct {
replaying func() []int
broadcastOn bool
broadcastProblem func() string
silentPeers func() ([]string, time.Duration)
}

func registerHealthChecks(srv *api.Server, src healthSources) {
Expand All @@ -24,6 +26,7 @@ func registerHealthChecks(srv *api.Server, src healthSources) {
check("journal", func() api.ComponentHealth { return journalHealth(src.journal()) })
check("recovery", func() api.ComponentHealth { return recoveryHealth(src.replaying()) })
check("broadcast", func() api.ComponentHealth { return broadcastHealth(src.broadcastOn, src.broadcastProblem()) })
check("broadcast-peers", func() api.ComponentHealth { return silentPeersHealth(src.silentPeers()) })
}

func healthy(message string) api.ComponentHealth {
Expand Down Expand Up @@ -77,3 +80,10 @@ func broadcastHealth(on bool, problem string) api.ComponentHealth {
}
return healthy("")
}

func silentPeersHealth(names []string, quiet time.Duration) api.ComponentHealth {
if len(names) == 0 {
return healthy("")
}
return degraded("no broadcast heard from %s for over %v, though the registry shows them mounted; this node sees their changes only after -cache-ttl. Check that multicast reaches this node, and that they run with -broadcast and the same -mcast-group and -mcast-port", strings.Join(names, ", "), quiet)
}
14 changes: 12 additions & 2 deletions cmd/goclusterfs-mount/health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"
"syscall"
"testing"
"time"

"github.com/supporttools/GoClusterFS/internal/api"
)
Expand Down Expand Up @@ -37,6 +38,11 @@ func TestHealthChecksSeparateDegradedFromDown(t *testing.T) {
s.broadcastProblem = func() string { return "multicast is unavailable and no unicast peers are known" }
return s
}(), want: api.HealthStatusDegraded, code: http.StatusOK, problem: "broadcast"},
{name: "a mounted peer is silent", src: func() healthSources {
s := fineSources()
s.silentPeers = func() ([]string, time.Duration) { return []string{"node3 (1a2b3c4d)"}, 90 * time.Second }
return s
}(), want: api.HealthStatusDegraded, code: http.StatusOK, problem: "broadcast-peers"},
{name: "device failing", src: func() healthSources {
s := fineSources()
s.deviceError = func() error { return syscall.EIO }
Expand All @@ -63,8 +69,8 @@ func TestHealthChecksSeparateDegradedFromDown(t *testing.T) {
if resp.Status != tt.want {
t.Errorf("status = %s, want %s: %+v", resp.Status, tt.want, resp.Checks)
}
if len(resp.Checks) != 4 {
t.Errorf("checks = %v, want device, journal, recovery and broadcast", resp.Checks)
if len(resp.Checks) != 5 {
t.Errorf("checks = %v, want device, journal, recovery, broadcast and broadcast-peers", resp.Checks)
}
for name, check := range resp.Checks {
if (name == tt.problem) != (check.Status != api.HealthStatusOK) {
Expand Down Expand Up @@ -101,6 +107,9 @@ func TestHealthMessagesNameTheSlotsAndTheError(t *testing.T) {
if got := journalHealth(&api.JournalStatsResponse{State: "recovering"}); got.Status != api.HealthStatusDegraded {
t.Errorf("recovering journal = %+v, want degraded", got)
}
if msg := silentPeersHealth([]string{"a (01)", "b (02)"}, 90*time.Second).Message; !strings.Contains(msg, "a (01), b (02)") || !strings.Contains(msg, "1m30s") {
t.Errorf("silent peers message = %q", msg)
}
}

func fineSources() healthSources {
Expand All @@ -110,5 +119,6 @@ func fineSources() healthSources {
replaying: func() []int { return nil },
broadcastOn: true,
broadcastProblem: func() string { return "" },
silentPeers: func() ([]string, time.Duration) { return nil, 90 * time.Second },
}
}
8 changes: 7 additions & 1 deletion cmd/goclusterfs-mount/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ func main() {
broadcastSecret = flag.String("broadcast-secret", "", "Secret that signs broadcast messages, the same on every node; visible to local users in ps. Empty leaves them unauthenticated: any host that can reach the UDP ports can make this node drop its caches")
broadcastIface = flag.String("broadcast-iface", "", "Network interface for multicast; empty lets the kernel choose")
broadcastPeers = flag.Int("broadcast-max-peers", broadcast.DefaultMaxPeers, "Most nodes the broadcast peer table and sequence tracking hold")
broadcastBeat = flag.Duration("broadcast-heartbeat", 30*time.Second, "How often this node sends a broadcast heartbeat; a mounted node not heard for 3 of these is reported")

// TRIM configuration
trimEnabled = flag.Bool("trim", true, "Discard freed blocks on the device (BLKDISCARD) in batches; does nothing on image files")
Expand Down Expand Up @@ -193,11 +194,14 @@ func main() {
UnicastPort: *unicastPort,
MulticastTTL: 1, // Same subnet
MaxMulticastFailures: 3,
HeartbeatInterval: 30 * time.Second,
HeartbeatInterval: *broadcastBeat,
Secret: *broadcastSecret,
Interface: *broadcastIface,
MaxPeers: *broadcastPeers,
}
if *broadcastBeat <= 0 {
log.Fatalf("-broadcast-heartbeat must be positive, got %v", *broadcastBeat)
}
if *broadcastPeers < 1 {
log.Fatalf("-broadcast-max-peers must be at least 1, got %d", *broadcastPeers)
}
Expand Down Expand Up @@ -408,6 +412,7 @@ func main() {
if err := cacheMgr.StartBroadcast(ctx, nodeUUID, uint32(slotIdx)); err != nil {
broadcastFailed(err)
}
cacheMgr.WatchBroadcastPeers(expectedPeers(nodeReg))
log.Printf("Broadcast: enabled on %s:%d, unicast port %d (secret=%v)",
*multicastGroup, *multicastPort, *unicastPort, *broadcastSecret != "")
} else {
Expand Down Expand Up @@ -538,6 +543,7 @@ func main() {
replaying: nodeReg.Replaying,
broadcastOn: cacheMgr.BroadcastEnabled(),
broadcastProblem: cacheMgr.BroadcastProblem,
silentPeers: cacheMgr.SilentBroadcastPeers,
})
var snaps fileops.Snapshots
if gfs.SnapshotRegistry() != nil {
Expand Down
29 changes: 29 additions & 0 deletions cmd/goclusterfs-mount/peers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package main

import (
"fmt"
"strings"

"github.com/supporttools/GoClusterFS/internal/cache/broadcast"
"github.com/supporttools/GoClusterFS/internal/node"
)

// expectedPeers lists the nodes the registry shows mounted, which should all
// be sending this node broadcasts.
func expectedPeers(reg *node.Registry) func() ([]broadcast.ExpectedPeer, error) {
return func() ([]broadcast.ExpectedPeer, error) {
slots, err := reg.GetActiveNodes()
if err != nil {
return nil, err
}
peers := make([]broadcast.ExpectedPeer, 0, len(slots))
for _, s := range slots {
host := strings.TrimRight(string(s.Hostname[:]), "\x00")
peers = append(peers, broadcast.ExpectedPeer{
UUID: s.UUID,
Name: fmt.Sprintf("%s (%x)", host, s.UUID[:4]),
})
}
return peers, nil
}
}
10 changes: 6 additions & 4 deletions docs/API-CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For
| `-broadcast-secret` | string | (empty) | Secret that signs broadcast messages, the same on every node; visible to local users in `ps`. Empty leaves them unauthenticated |
| `-broadcast-iface` | string | (empty) | Network interface for multicast; empty lets the kernel choose |
| `-broadcast-max-peers` | int | `256` | Most nodes the unicast peer table and the lost-message tracking hold. When either is full, the node heard from longest ago is dropped |
| `-broadcast-heartbeat` | duration | `30s` | How often a node sends a broadcast heartbeat. A node the registry shows mounted but not heard on broadcast for 3 of these is reported in the log, on `/health` and on `/metrics` |

### TRIM Flags

Expand Down Expand Up @@ -105,7 +106,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For
| `-api-rate-limit` | float | `0` (off) | Requests per second per client IP, with bursts of twice that, up to 1,000,000. Over the limit, the API answers 429 |
| `-dashboard` | bool | `true` | Serve the [web dashboard](#web-dashboard) under `/ui/` |

The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status`, `GET /api/v1/stats/...`, the [web dashboard](#web-dashboard) under `/ui/` and, with a token, the [write endpoints](#http-api-write-endpoints). [HTTP API Statistics](#http-api-statistics) lists what `/metrics`, `/api/v1/status` and `/api/v1/stats/...` return. `/health` checks the device, journal, journal recovery and broadcast; see [HTTP API Health and Shutdown](#http-api-health-and-shutdown). With a token set, `/api/v1/...` requests need `Authorization: Bearer <token>`. `/health`, `/metrics` and the dashboard's files under `/ui/` never need the token.
The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status`, `GET /api/v1/stats/...`, the [web dashboard](#web-dashboard) under `/ui/` and, with a token, the [write endpoints](#http-api-write-endpoints). [HTTP API Statistics](#http-api-statistics) lists what `/metrics`, `/api/v1/status` and `/api/v1/stats/...` return. `/health` checks the device, journal, journal recovery, broadcast and broadcast peers; see [HTTP API Health and Shutdown](#http-api-health-and-shutdown). With a token set, `/api/v1/...` requests need `Authorization: Bearer <token>`. `/health`, `/metrics` and the dashboard's files under `/ui/` never need the token.

Without a token, the API answers only requests whose `Host` header is `localhost`, a `127.x` address or `[::1]`, so a web page cannot reach it through DNS rebinding, and mount refuses `-api-cors '*'`. List the allowed origins instead. The dashboard at `/ui/` shares the API's origin and needs no CORS.

Expand Down Expand Up @@ -159,28 +160,29 @@ Mount copies its managers' counters every second. While requests read capacity o
|----------|---------|
| `/api/v1/status` | Version, uptime, mount and device paths, and `capacity`: total, used and free blocks and bytes of the data region, for the whole volume |
| `/api/v1/stats` | `cache` and `journal`, as below |
| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `capacity` (the limit), `hits`, `misses` and `hit_rate` (0 to 1). `lock_hint_cache` is always 0: locks no longer use a hint cache. With broadcast on, `broadcast` has `peers` (unicast peers), `senders` (nodes whose message numbers this node tracks) and `max_peers` `dir_index` has `dirs` and `entries` (names) held, `limit`, `hits` (lookups an index answered), `misses`, `rebuilds` (full reads of a directory) and `hit_rate`. |
| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `capacity` (the limit), `hits`, `misses` and `hit_rate` (0 to 1). `lock_hint_cache` is always 0: locks no longer use a hint cache. With broadcast on, `broadcast` has `peers` (unicast peers), `senders` (nodes whose message numbers this node tracks), `silent_peers` (mounted nodes not heard on broadcast for 3 heartbeat intervals) and `max_peers` `dir_index` has `dirs` and `entries` (names) held, `limit`, `hits` (lookups an index answered), `misses`, `rebuilds` (full reads of a directory) and `hit_rate`. |
| `/api/v1/stats/journal` | This node's journal: `state` (`clean` or `dirty`), `head_offset` and `tail_offset` in the ring, `available_space` in bytes, `active_txn_count`, `commit_count`, `bytes_written`, `last_checkpoint_seq`, `batch_sync_count`. 404 with `-journal=false` |
| `/api/v1/stats/trim` | `trim_enabled`, false with `-trim=false` or after the device refused a discard as unsupported (image files always do). With `-trim`, also `trim_operations` (discard calls), `trim_blocks` and `trim_bytes` discarded, `pending_trim_blocks`, `trim_errors` (failed ranges, plus flushes skipped because the freed blocks could not be reserved), `last_trim_at` and `batch_size` |
| `/api/v1/stats/lock` | `active_locks` (locks this node holds), `pending_locks` (lock requests that found the lock taken and are retrying), `hint_cache_hits`, `hint_cache_misses` and `hint_cache_hit_rate`, which are always 0, and `types`: per lock type (`bitmap`, `refcount`, `inode`, ...), the `acquires` served, the `blocked` acquires whose first try found the lock taken, and their `wait_seconds` |
| `/api/v1/stats/cluster` | This node's `node_id` (UUID in hex), `node_address` (hostname), `state`, `last_heartbeat_at` and `joined_at`; `cluster_size` (registered nodes); `healthy_nodes` (active with a heartbeat under 10 seconds old); and `peers`, each with `node_id`, `address` (hostname), `state` and `last_seen`. A state is `active`, `leaving`, `dead`, or `stale` when the heartbeat is over 10 seconds old and no node has declared it dead yet. `role` is always `peer`: there is no leader |
| `/api/v1/stats/sfb` | 404, because SFB containers are disabled |
| `/api/v1/stats/all` | `collected_at` and every section above except `sfb` |

`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics.
`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders`, `broadcast_silent_peers` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics.

---

## HTTP API Health and Shutdown

`GET /health` runs four checks and reports the worst of them:
`GET /health` runs five checks and reports the worst of them:

| Check | `ok` | `degraded` | `unhealthy` |
|-------|------|------------|-------------|
| `device` | The last device read, write or sync worked | | The last device read, write or sync failed |
| `journal` | Clean or dirty, or off with `-journal=false` | This node's journal is being replayed | This node's journal is corrupt |
| `recovery` | No replay is running | This node replays the journal of a crashed node | |
| `broadcast` | Working, or off with `-broadcast=false` | The last invalidation send failed, or multicast is unavailable and no unicast peers are known | |
| `broadcast-peers` | This node hears every node the registry shows mounted, or broadcast is off | A mounted node has not been heard on broadcast for 3 `-broadcast-heartbeat` intervals: multicast may not reach this node, or that node runs with `-broadcast=false` or another `-mcast-group` or `-mcast-port`. This node sees that node's changes in directories only after `-cache-ttl` | |

`ok` and `degraded` answer 200; `unhealthy` answers 503. Degraded means the mount works with a problem: during a replay, files the crashed node had locked wait until it ends, and with broadcast down, peers see this node's changes only after `-cache-ttl`. Unhealthy means file operations fail. A mount that is not running refuses the connection. A check that is not `ok` has a `message` that says what to do. `gfsctl health` exits 0, 1 or 4 for the three states.

Expand Down
1 change: 1 addition & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Broadcast stats: sent=..., received=..., duplicates=..., lost=...
| `Failed to start cache broadcast: ...` | The mount stops before it touches the device. See [Mount Fails: "Failed to start cache broadcast"](TROUBLESHOOTING.md#mount-fails-failed-to-start-cache-broadcast). |
| `Warning: Failed to start API server` | The API port is taken. The mount runs without the API, `/metrics`, `gfsctl` and the dashboard. |
| `[broadcast] missed N messages from ...` | This node lost broadcast messages, usually because its receive buffer filled during a burst of changes on another node. It dropped all cached inodes and extents and rereads them from the device. Logged at most every 10 seconds. If it repeats, raise `net.core.rmem_max` to at least 1 MB (`sysctl -w net.core.rmem_max=1048576`) and remount. |
| `[broadcast] WARN: no broadcast heard from node ...` | The registry shows that node mounted, but this node has heard none of its broadcasts for 3 `-broadcast-heartbeat` intervals (90 seconds by default). A network that drops multicast causes this, as does a node with `-broadcast=false` or another `-mcast-group` or `-mcast-port`. Until it is fixed, this node sees that node's directory changes only after `-cache-ttl`. `/health` reports it as `broadcast-peers`, and the mount logs `heard from node ... again` when it ends. |
| `lock.Acquire: ... acquired (retries=N, dur=...)` | Lock waits. High `retries` or `dur` means nodes contend for the same files or directories. |
| `[WARN] lock: ... which holds it and is failing or being recovered` | A lock wait for a failed node. It repeats every 30 seconds and ends when that node's journal replay ends. |
| `[WARN] lock: gave up waiting for ...` | A lock wait gave up: after 30 seconds (`DefaultTimeout`) on a live holder, after 30 seconds on a failed holder while the operation held another lock, at once when the holder's replay failed, or because a background task stopped at unmount. The line names the holder. A file operation that was waiting fails with EIO; during block allocation it fails with ENOSPC. See [Distributed Lock Limits](#distributed-lock-limits). |
Expand Down
Loading
Loading