From 3ddd6547977e7a0b2aec07c2491d3a8e6dd088fb Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 13:28:53 -0600 Subject: [PATCH 1/2] feat(broadcast): report mounted nodes this node hears no broadcasts from A network that drops multicast was invisible: a node that hears nothing from a peer finds no gap in its sequence numbers, so #96's loss check never fires, and only directory lookups lagging by -cache-ttl showed it. Every -broadcast-heartbeat interval (30s by default, now a flag), the broadcaster compares the nodes the registry shows mounted with the ones it has heard. A node silent for 3 intervals since it mounted, or since this node started, gets a WARN naming it, a degraded broadcast-peers check on /health, and a count in goclusterfs_broadcast_silent_peers. Hearing it again logs that and clears it. The Docker nodes run with a 2s heartbeat, so the new Docker test sees node 2 report its three peers within seconds of dropping multicast and recover after the fault is healed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + cmd/goclusterfs-mount/health.go | 10 +++ cmd/goclusterfs-mount/health_test.go | 14 ++- cmd/goclusterfs-mount/main.go | 8 +- cmd/goclusterfs-mount/peers.go | 31 +++++++ docs/API-CONFIG.md | 10 ++- docs/OPERATIONS.md | 1 + internal/api/prometheus.go | 8 ++ internal/api/prometheus_test.go | 3 +- internal/api/types.go | 7 +- internal/cache/broadcast/receiver.go | 14 ++- internal/cache/broadcast/silence.go | 108 +++++++++++++++++++++++ internal/cache/broadcast/silence_test.go | 77 ++++++++++++++++ internal/cache/cache.go | 19 ++++ internal/stats/convert.go | 2 +- tests/docker/docker-compose.test.yml | 5 ++ tests/docker/scripts/entrypoint.sh | 3 + tests/docker/scripts/run-tests.sh | 57 ++++++++++++ tests/integration/broadcast_test.go | 57 ++++++++++++ tests/test-manifest.yaml | 16 ++++ 20 files changed, 438 insertions(+), 13 deletions(-) create mode 100644 cmd/goclusterfs-mount/peers.go create mode 100644 internal/cache/broadcast/silence.go create mode 100644 internal/cache/broadcast/silence_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f971167c..68e78808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - A shutdown step that takes longer than 2 seconds logs its name and duration, and a shutdown still running after 8 seconds logs every goroutine's stack. - `-cache-inodes`, `-cache-extents`, `-cache-negative` and `-broadcast-max-peers` set the cache and broadcast table sizes, and `/metrics` reports each limit next to its size. +- A node that hears no broadcasts from a node the registry shows mounted, for 3 `-broadcast-heartbeat` intervals (90 seconds by default), logs a warning naming it, reports `broadcast-peers` as degraded on `/health`, and counts it in `goclusterfs_broadcast_silent_peers`. A network that silently drops multicast was invisible before. ### Changed diff --git a/cmd/goclusterfs-mount/health.go b/cmd/goclusterfs-mount/health.go index 7bc02a16..02224f02 100644 --- a/cmd/goclusterfs-mount/health.go +++ b/cmd/goclusterfs-mount/health.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/supporttools/GoClusterFS/internal/api" ) @@ -14,6 +15,7 @@ type healthSources struct { replaying func() []int broadcastOn bool broadcastProblem func() string + silentPeers func() ([]string, time.Duration) } func registerHealthChecks(srv *api.Server, src healthSources) { @@ -24,6 +26,7 @@ func registerHealthChecks(srv *api.Server, src healthSources) { check("journal", func() api.ComponentHealth { return journalHealth(src.journal()) }) check("recovery", func() api.ComponentHealth { return recoveryHealth(src.replaying()) }) check("broadcast", func() api.ComponentHealth { return broadcastHealth(src.broadcastOn, src.broadcastProblem()) }) + check("broadcast-peers", func() api.ComponentHealth { return silentPeersHealth(src.silentPeers()) }) } func healthy(message string) api.ComponentHealth { @@ -77,3 +80,10 @@ func broadcastHealth(on bool, problem string) api.ComponentHealth { } return healthy("") } + +func silentPeersHealth(names []string, quiet time.Duration) api.ComponentHealth { + if len(names) == 0 { + return healthy("") + } + return degraded("no broadcast heard from %s for over %v, though the registry shows them mounted; this node sees their changes only after -cache-ttl. Check that multicast reaches this node, and that they run with -broadcast and the same -mcast-group and -mcast-port", strings.Join(names, ", "), quiet) +} diff --git a/cmd/goclusterfs-mount/health_test.go b/cmd/goclusterfs-mount/health_test.go index 746acbbd..4e042a3c 100644 --- a/cmd/goclusterfs-mount/health_test.go +++ b/cmd/goclusterfs-mount/health_test.go @@ -8,6 +8,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/supporttools/GoClusterFS/internal/api" ) @@ -37,6 +38,11 @@ func TestHealthChecksSeparateDegradedFromDown(t *testing.T) { s.broadcastProblem = func() string { return "multicast is unavailable and no unicast peers are known" } return s }(), want: api.HealthStatusDegraded, code: http.StatusOK, problem: "broadcast"}, + {name: "a mounted peer is silent", src: func() healthSources { + s := fineSources() + s.silentPeers = func() ([]string, time.Duration) { return []string{"node3 (1a2b3c4d)"}, 90 * time.Second } + return s + }(), want: api.HealthStatusDegraded, code: http.StatusOK, problem: "broadcast-peers"}, {name: "device failing", src: func() healthSources { s := fineSources() s.deviceError = func() error { return syscall.EIO } @@ -63,8 +69,8 @@ func TestHealthChecksSeparateDegradedFromDown(t *testing.T) { if resp.Status != tt.want { t.Errorf("status = %s, want %s: %+v", resp.Status, tt.want, resp.Checks) } - if len(resp.Checks) != 4 { - t.Errorf("checks = %v, want device, journal, recovery and broadcast", resp.Checks) + if len(resp.Checks) != 5 { + t.Errorf("checks = %v, want device, journal, recovery, broadcast and broadcast-peers", resp.Checks) } for name, check := range resp.Checks { if (name == tt.problem) != (check.Status != api.HealthStatusOK) { @@ -101,6 +107,9 @@ func TestHealthMessagesNameTheSlotsAndTheError(t *testing.T) { if got := journalHealth(&api.JournalStatsResponse{State: "recovering"}); got.Status != api.HealthStatusDegraded { t.Errorf("recovering journal = %+v, want degraded", got) } + if msg := silentPeersHealth([]string{"a (01)", "b (02)"}, 90*time.Second).Message; !strings.Contains(msg, "a (01), b (02)") || !strings.Contains(msg, "1m30s") { + t.Errorf("silent peers message = %q", msg) + } } func fineSources() healthSources { @@ -110,5 +119,6 @@ func fineSources() healthSources { replaying: func() []int { return nil }, broadcastOn: true, broadcastProblem: func() string { return "" }, + silentPeers: func() ([]string, time.Duration) { return nil, 90 * time.Second }, } } diff --git a/cmd/goclusterfs-mount/main.go b/cmd/goclusterfs-mount/main.go index 08ee9578..a12c339c 100644 --- a/cmd/goclusterfs-mount/main.go +++ b/cmd/goclusterfs-mount/main.go @@ -108,6 +108,7 @@ func main() { broadcastSecret = flag.String("broadcast-secret", "", "Secret that signs broadcast messages, the same on every node; visible to local users in ps. Empty leaves them unauthenticated: any host that can reach the UDP ports can make this node drop its caches") broadcastIface = flag.String("broadcast-iface", "", "Network interface for multicast; empty lets the kernel choose") broadcastPeers = flag.Int("broadcast-max-peers", broadcast.DefaultMaxPeers, "Most nodes the broadcast peer table and sequence tracking hold") + broadcastBeat = flag.Duration("broadcast-heartbeat", 30*time.Second, "How often this node sends a broadcast heartbeat; a mounted node not heard for 3 of these is reported") // TRIM configuration trimEnabled = flag.Bool("trim", true, "Discard freed blocks on the device (BLKDISCARD) in batches; does nothing on image files") @@ -193,11 +194,14 @@ func main() { UnicastPort: *unicastPort, MulticastTTL: 1, // Same subnet MaxMulticastFailures: 3, - HeartbeatInterval: 30 * time.Second, + HeartbeatInterval: *broadcastBeat, Secret: *broadcastSecret, Interface: *broadcastIface, MaxPeers: *broadcastPeers, } + if *broadcastBeat <= 0 { + log.Fatalf("-broadcast-heartbeat must be positive, got %v", *broadcastBeat) + } if *broadcastPeers < 1 { log.Fatalf("-broadcast-max-peers must be at least 1, got %d", *broadcastPeers) } @@ -408,6 +412,7 @@ func main() { if err := cacheMgr.StartBroadcast(ctx, nodeUUID, uint32(slotIdx)); err != nil { broadcastFailed(err) } + cacheMgr.WatchBroadcastPeers(expectedPeers(nodeReg)) log.Printf("Broadcast: enabled on %s:%d, unicast port %d (secret=%v)", *multicastGroup, *multicastPort, *unicastPort, *broadcastSecret != "") } else { @@ -538,6 +543,7 @@ func main() { replaying: nodeReg.Replaying, broadcastOn: cacheMgr.BroadcastEnabled(), broadcastProblem: cacheMgr.BroadcastProblem, + silentPeers: cacheMgr.SilentBroadcastPeers, }) var snaps fileops.Snapshots if gfs.SnapshotRegistry() != nil { diff --git a/cmd/goclusterfs-mount/peers.go b/cmd/goclusterfs-mount/peers.go new file mode 100644 index 00000000..a870b812 --- /dev/null +++ b/cmd/goclusterfs-mount/peers.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/supporttools/GoClusterFS/internal/cache/broadcast" + "github.com/supporttools/GoClusterFS/internal/node" +) + +// expectedPeers lists the nodes the registry shows mounted, which should all +// be sending this node broadcasts. +func expectedPeers(reg *node.Registry) func() ([]broadcast.ExpectedPeer, error) { + return func() ([]broadcast.ExpectedPeer, error) { + slots, err := reg.GetActiveNodes() + if err != nil { + return nil, err + } + peers := make([]broadcast.ExpectedPeer, 0, len(slots)) + for _, s := range slots { + host := strings.TrimRight(string(s.Hostname[:]), "\x00") + peers = append(peers, broadcast.ExpectedPeer{ + UUID: s.UUID, + Name: fmt.Sprintf("%s (%x)", host, s.UUID[:4]), + Mounted: time.Unix(s.MountTime, 0), + }) + } + return peers, nil + } +} diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index c529ec55..5082144f 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -77,6 +77,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For | `-broadcast-secret` | string | (empty) | Secret that signs broadcast messages, the same on every node; visible to local users in `ps`. Empty leaves them unauthenticated | | `-broadcast-iface` | string | (empty) | Network interface for multicast; empty lets the kernel choose | | `-broadcast-max-peers` | int | `256` | Most nodes the unicast peer table and the lost-message tracking hold. When either is full, the node heard from longest ago is dropped | +| `-broadcast-heartbeat` | duration | `30s` | How often a node sends a broadcast heartbeat. A node the registry shows mounted but not heard on broadcast for 3 of these is reported in the log, on `/health` and on `/metrics` | ### TRIM Flags @@ -105,7 +106,7 @@ The kernel caches file attributes and directory entries for `-attr-timeout`. For | `-api-rate-limit` | float | `0` (off) | Requests per second per client IP, with bursts of twice that, up to 1,000,000. Over the limit, the API answers 429 | | `-dashboard` | bool | `true` | Serve the [web dashboard](#web-dashboard) under `/ui/` | -The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status`, `GET /api/v1/stats/...`, the [web dashboard](#web-dashboard) under `/ui/` and, with a token, the [write endpoints](#http-api-write-endpoints). [HTTP API Statistics](#http-api-statistics) lists what `/metrics`, `/api/v1/status` and `/api/v1/stats/...` return. `/health` checks the device, journal, journal recovery and broadcast; see [HTTP API Health and Shutdown](#http-api-health-and-shutdown). With a token set, `/api/v1/...` requests need `Authorization: Bearer `. `/health`, `/metrics` and the dashboard's files under `/ui/` never need the token. +The API serves `GET /health`, `GET /metrics`, `GET /api/v1/status`, `GET /api/v1/stats/...`, the [web dashboard](#web-dashboard) under `/ui/` and, with a token, the [write endpoints](#http-api-write-endpoints). [HTTP API Statistics](#http-api-statistics) lists what `/metrics`, `/api/v1/status` and `/api/v1/stats/...` return. `/health` checks the device, journal, journal recovery, broadcast and broadcast peers; see [HTTP API Health and Shutdown](#http-api-health-and-shutdown). With a token set, `/api/v1/...` requests need `Authorization: Bearer `. `/health`, `/metrics` and the dashboard's files under `/ui/` never need the token. Without a token, the API answers only requests whose `Host` header is `localhost`, a `127.x` address or `[::1]`, so a web page cannot reach it through DNS rebinding, and mount refuses `-api-cors '*'`. List the allowed origins instead. The dashboard at `/ui/` shares the API's origin and needs no CORS. @@ -159,7 +160,7 @@ Mount copies its managers' counters every second. While requests read capacity o |----------|---------| | `/api/v1/status` | Version, uptime, mount and device paths, and `capacity`: total, used and free blocks and bytes of the data region, for the whole volume | | `/api/v1/stats` | `cache` and `journal`, as below | -| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `capacity` (the limit), `hits`, `misses` and `hit_rate` (0 to 1). `lock_hint_cache` is always 0: locks no longer use a hint cache. With broadcast on, `broadcast` has `peers` (unicast peers), `senders` (nodes whose message numbers this node tracks) and `max_peers` `dir_index` has `dirs` and `entries` (names) held, `limit`, `hits` (lookups an index answered), `misses`, `rebuilds` (full reads of a directory) and `hit_rate`. | +| `/api/v1/stats/cache` | `inode_cache`, `extent_cache`, `negative_cache` and `lock_hint_cache`, each with `size` (entries), `capacity` (the limit), `hits`, `misses` and `hit_rate` (0 to 1). `lock_hint_cache` is always 0: locks no longer use a hint cache. With broadcast on, `broadcast` has `peers` (unicast peers), `senders` (nodes whose message numbers this node tracks), `silent_peers` (mounted nodes not heard on broadcast for 3 heartbeat intervals) and `max_peers` `dir_index` has `dirs` and `entries` (names) held, `limit`, `hits` (lookups an index answered), `misses`, `rebuilds` (full reads of a directory) and `hit_rate`. | | `/api/v1/stats/journal` | This node's journal: `state` (`clean` or `dirty`), `head_offset` and `tail_offset` in the ring, `available_space` in bytes, `active_txn_count`, `commit_count`, `bytes_written`, `last_checkpoint_seq`, `batch_sync_count`. 404 with `-journal=false` | | `/api/v1/stats/trim` | `trim_enabled`, false with `-trim=false` or after the device refused a discard as unsupported (image files always do). With `-trim`, also `trim_operations` (discard calls), `trim_blocks` and `trim_bytes` discarded, `pending_trim_blocks`, `trim_errors` (failed ranges, plus flushes skipped because the freed blocks could not be reserved), `last_trim_at` and `batch_size` | | `/api/v1/stats/lock` | `active_locks` (locks this node holds), `pending_locks` (lock requests that found the lock taken and are retrying), `hint_cache_hits`, `hint_cache_misses` and `hint_cache_hit_rate`, which are always 0, and `types`: per lock type (`bitmap`, `refcount`, `inode`, ...), the `acquires` served, the `blocked` acquires whose first try found the lock taken, and their `wait_seconds` | @@ -167,13 +168,13 @@ Mount copies its managers' counters every second. While requests read capacity o | `/api/v1/stats/sfb` | 404, because SFB containers are disabled | | `/api/v1/stats/all` | `collected_at` and every section above except `sfb` | -`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics. +`/metrics` exports the same counters under the `goclusterfs_` prefix: `cache_*` (including `cache_inode_capacity`, `cache_extent_capacity` and `cache_negative_capacity`), `dir_index_*`, `broadcast_peers`, `broadcast_senders`, `broadcast_silent_peers` and `broadcast_max_peers` (with broadcast on), `journal_*`, `trim_*`, `lock_*` (including `lock_acquires_total`, `lock_blocked_total` and `lock_wait_seconds_total`, labeled by lock `type`), `reserve_blocks`, `reserve_refills_total`, `reserve_returns_total`, `reserve_peer_waits_total` and `reserve_peer_wait_seconds_total` (with block reserves on), `cluster_size`, `cluster_healthy_nodes` and `cluster_is_leader` (always 0). `goclusterfs_journal_state` is 0 for clean, 1 for dirty, 2 for recovering and 3 for corrupt. `goclusterfs_fs_size_bytes` and `goclusterfs_fs_free_bytes` give the data region's capacity. There are no `sfb_*` metrics. --- ## HTTP API Health and Shutdown -`GET /health` runs four checks and reports the worst of them: +`GET /health` runs five checks and reports the worst of them: | Check | `ok` | `degraded` | `unhealthy` | |-------|------|------------|-------------| @@ -181,6 +182,7 @@ Mount copies its managers' counters every second. While requests read capacity o | `journal` | Clean or dirty, or off with `-journal=false` | This node's journal is being replayed | This node's journal is corrupt | | `recovery` | No replay is running | This node replays the journal of a crashed node | | | `broadcast` | Working, or off with `-broadcast=false` | The last invalidation send failed, or multicast is unavailable and no unicast peers are known | | +| `broadcast-peers` | This node hears every node the registry shows mounted, or broadcast is off | A mounted node has not been heard on broadcast for 3 `-broadcast-heartbeat` intervals: multicast may not reach this node, or that node runs with `-broadcast=false` or another `-mcast-group` or `-mcast-port`. This node sees that node's changes in directories only after `-cache-ttl` | | `ok` and `degraded` answer 200; `unhealthy` answers 503. Degraded means the mount works with a problem: during a replay, files the crashed node had locked wait until it ends, and with broadcast down, peers see this node's changes only after `-cache-ttl`. Unhealthy means file operations fail. A mount that is not running refuses the connection. A check that is not `ok` has a `message` that says what to do. `gfsctl health` exits 0, 1 or 4 for the three states. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 1941afe4..021e8e65 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -49,6 +49,7 @@ Broadcast stats: sent=..., received=..., duplicates=..., lost=... | `Failed to start cache broadcast: ...` | The mount stops before it touches the device. See [Mount Fails: "Failed to start cache broadcast"](TROUBLESHOOTING.md#mount-fails-failed-to-start-cache-broadcast). | | `Warning: Failed to start API server` | The API port is taken. The mount runs without the API, `/metrics`, `gfsctl` and the dashboard. | | `[broadcast] missed N messages from ...` | This node lost broadcast messages, usually because its receive buffer filled during a burst of changes on another node. It dropped all cached inodes and extents and rereads them from the device. Logged at most every 10 seconds. If it repeats, raise `net.core.rmem_max` to at least 1 MB (`sysctl -w net.core.rmem_max=1048576`) and remount. | +| `[broadcast] WARN: no broadcast heard from node ...` | The registry shows that node mounted, but this node has heard none of its broadcasts for 3 `-broadcast-heartbeat` intervals (90 seconds by default). A network that drops multicast causes this, as does a node with `-broadcast=false` or another `-mcast-group` or `-mcast-port`. Until it is fixed, this node sees that node's directory changes only after `-cache-ttl`. `/health` reports it as `broadcast-peers`, and the mount logs `heard from node ... again` when it ends. | | `lock.Acquire: ... acquired (retries=N, dur=...)` | Lock waits. High `retries` or `dur` means nodes contend for the same files or directories. | | `[WARN] lock: ... which holds it and is failing or being recovered` | A lock wait for a failed node. It repeats every 30 seconds and ends when that node's journal replay ends. | | `[WARN] lock: gave up waiting for ...` | A lock wait gave up: after 30 seconds (`DefaultTimeout`) on a live holder, after 30 seconds on a failed holder while the operation held another lock, at once when the holder's replay failed, or because a background task stopped at unmount. The line names the holder. A file operation that was waiting fails with EIO; during block allocation it fails with ENOSPC. See [Distributed Lock Limits](#distributed-lock-limits). | diff --git a/internal/api/prometheus.go b/internal/api/prometheus.go index 0750034f..a19a7c25 100644 --- a/internal/api/prometheus.go +++ b/internal/api/prometheus.go @@ -45,6 +45,7 @@ type fsCollector struct { broadcastPeersDesc *prometheus.Desc broadcastSendersDesc *prometheus.Desc broadcastMaxPeersDesc *prometheus.Desc + broadcastSilentPeersDesc *prometheus.Desc cacheInodeSizeDesc *prometheus.Desc cacheInodeHitsDesc *prometheus.Desc cacheInodeMissesDesc *prometheus.Desc @@ -161,6 +162,11 @@ func newFSCollector(s *Server) *fsCollector { "Nodes whose broadcast sequence numbers this node tracks", nil, nil, ), + broadcastSilentPeersDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "broadcast", "silent_peers"), + "Mounted nodes this node has not heard on broadcast for 3 heartbeat intervals", + nil, nil, + ), broadcastMaxPeersDesc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "broadcast", "max_peers"), "Most entries each broadcast table holds (-broadcast-max-peers)", @@ -516,6 +522,7 @@ func (c *fsCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.broadcastPeersDesc ch <- c.broadcastSendersDesc ch <- c.broadcastMaxPeersDesc + ch <- c.broadcastSilentPeersDesc ch <- c.cacheInodeSizeDesc ch <- c.cacheInodeHitsDesc ch <- c.cacheInodeMissesDesc @@ -658,6 +665,7 @@ func (c *fsCollector) collectCacheStats(ch chan<- prometheus.Metric, stats *Cach ch <- prometheus.MustNewConstMetric(c.broadcastPeersDesc, prometheus.GaugeValue, float64(b.Peers)) ch <- prometheus.MustNewConstMetric(c.broadcastSendersDesc, prometheus.GaugeValue, float64(b.Senders)) ch <- prometheus.MustNewConstMetric(c.broadcastMaxPeersDesc, prometheus.GaugeValue, float64(b.MaxPeers)) + ch <- prometheus.MustNewConstMetric(c.broadcastSilentPeersDesc, prometheus.GaugeValue, float64(b.SilentPeers)) } // Inode cache diff --git a/internal/api/prometheus_test.go b/internal/api/prometheus_test.go index 9fb402a2..b9382f9a 100644 --- a/internal/api/prometheus_test.go +++ b/internal/api/prometheus_test.go @@ -84,7 +84,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { Misses: 20, HitRate: 0.9, }, - Broadcast: &BroadcastTables{Peers: 3, Senders: 4, MaxPeers: 256}, + Broadcast: &BroadcastTables{Peers: 3, Senders: 4, SilentPeers: 1, MaxPeers: 256}, }, journal: &JournalStatsResponse{ State: "dirty", @@ -166,6 +166,7 @@ func TestMetricsEndpoint_WithStatsProvider(t *testing.T) { "goclusterfs_broadcast_peers 3", "goclusterfs_broadcast_senders 4", "goclusterfs_broadcast_max_peers 256", + "goclusterfs_broadcast_silent_peers 1", "goclusterfs_cache_inode_hits_total 800", "goclusterfs_cache_inode_misses_total 200", "goclusterfs_cache_inode_hit_rate 0.8", diff --git a/internal/api/types.go b/internal/api/types.go index 929c84ed..c5f28eb9 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -118,9 +118,10 @@ type DirIndexInfo struct { // BroadcastTables reports how full the broadcast peer tables are. type BroadcastTables struct { - Peers int `json:"peers"` - Senders int `json:"senders"` - MaxPeers int `json:"max_peers"` + Peers int `json:"peers"` + Senders int `json:"senders"` + SilentPeers int `json:"silent_peers"` + MaxPeers int `json:"max_peers"` } // JournalStatsResponse is returned by GET /api/v1/stats/journal. diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index a56ff7ca..3560224d 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -38,6 +38,11 @@ type Broadcaster struct { sendersMu sync.Mutex senders map[[16]byte]*senderSeq + started time.Time + expected atomic.Pointer[func() ([]ExpectedPeer, error)] + silentMu sync.Mutex + silent map[[16]byte]ExpectedPeer + lossLogMu sync.Mutex lossLoggedAt time.Time @@ -113,6 +118,7 @@ func (b *Broadcaster) Start(ctx context.Context) error { } b.ctx, b.cancel = context.WithCancel(ctx) + b.started = time.Now() // Start transport with our message handler if err := b.transport.Start(b.ctx, b.handleMessage); err != nil { @@ -121,9 +127,10 @@ func (b *Broadcaster) Start(ctx context.Context) error { } // Start background tasks - b.wg.Add(2) + b.wg.Add(3) go b.heartbeatLoop() go b.cleanupLoop() + go b.watchLoop() // Announce ourselves b.announcePresence() @@ -492,8 +499,12 @@ func (b *Broadcaster) Stats() BroadcasterStats { b.sendersMu.Lock() senders := len(b.senders) b.sendersMu.Unlock() + b.silentMu.Lock() + silent := b.silent + b.silentMu.Unlock() return BroadcasterStats{ TrackedSenders: senders, + SilentPeers: len(silent), MaxPeers: b.transport.config.maxPeers(), InvalidationsReceived: atomic.LoadUint64(&b.invalidationsReceived), InvalidationsSent: atomic.LoadUint64(&b.invalidationsSent), @@ -512,6 +523,7 @@ type BroadcasterStats struct { MessagesIgnored uint64 MessagesLost uint64 TrackedSenders int + SilentPeers int MaxPeers int Transport TransportStats } diff --git a/internal/cache/broadcast/silence.go b/internal/cache/broadcast/silence.go new file mode 100644 index 00000000..8b36c313 --- /dev/null +++ b/internal/cache/broadcast/silence.go @@ -0,0 +1,108 @@ +package broadcast + +import ( + "log" + "sort" + "time" +) + +// silentAfter is how many heartbeat intervals a node may go unheard before +// it counts as silent. +const silentAfter = 3 + +// ExpectedPeer is a node that should be sending this node broadcasts. +type ExpectedPeer struct { + UUID [16]byte + Name string + Mounted time.Time +} + +// WatchPeers has the broadcaster check, every heartbeat interval, that it +// hears from each node that expected lists. A node silent for silentAfter +// intervals since it mounted or this node started is reported: dropped +// multicast is otherwise invisible, because a node that hears nothing from a +// peer finds no gap in its sequence numbers either. +func (b *Broadcaster) WatchPeers(expected func() ([]ExpectedPeer, error)) { + b.expected.Store(&expected) +} + +// SilentPeers names the expected nodes not heard from for the returned time. +func (b *Broadcaster) SilentPeers() ([]string, time.Duration) { + b.silentMu.Lock() + defer b.silentMu.Unlock() + names := make([]string, 0, len(b.silent)) + for _, p := range b.silent { + names = append(names, p.Name) + } + sort.Strings(names) + return names, b.silenceThreshold() +} + +func (b *Broadcaster) silenceThreshold() time.Duration { + return silentAfter * b.transport.config.HeartbeatInterval +} + +func (b *Broadcaster) watchLoop() { + defer b.wg.Done() + + ticker := time.NewTicker(b.transport.config.HeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-b.ctx.Done(): + return + case now := <-ticker.C: + b.checkPeers(now) + } + } +} + +func (b *Broadcaster) checkPeers(now time.Time) { + expected := b.expected.Load() + if expected == nil { + return + } + peers, err := (*expected)() + if err != nil { + log.Printf("[broadcast] could not list the nodes that should be sending broadcasts: %v", err) + return + } + + quiet := b.silenceThreshold() + mounted := map[[16]byte]bool{} + silent := map[[16]byte]ExpectedPeer{} + b.sendersMu.Lock() + for _, p := range peers { + if p.UUID == b.localUUID { + continue + } + mounted[p.UUID] = true + last := b.started + if p.Mounted.After(last) { + last = p.Mounted + } + if s, ok := b.senders[p.UUID]; ok && s.heard.After(last) { + last = s.heard + } + if now.Sub(last) > quiet { + silent[p.UUID] = p + } + } + b.sendersMu.Unlock() + + b.silentMu.Lock() + defer b.silentMu.Unlock() + for id, p := range silent { + if _, ok := b.silent[id]; !ok { + log.Printf("[broadcast] WARN: no broadcast heard from node %s for over %v, though the registry shows it mounted; this node sees its changes only after -cache-ttl. Multicast may not reach this node, or that node runs with -broadcast=false or another -mcast-group or -mcast-port", + p.Name, quiet) + } + } + for id, p := range b.silent { + if _, ok := silent[id]; !ok && mounted[id] { + log.Printf("[broadcast] heard from node %s again", p.Name) + } + } + b.silent = silent +} diff --git a/internal/cache/broadcast/silence_test.go b/internal/cache/broadcast/silence_test.go new file mode 100644 index 00000000..73e8f0b5 --- /dev/null +++ b/internal/cache/broadcast/silence_test.go @@ -0,0 +1,77 @@ +package broadcast + +import ( + "errors" + "slices" + "testing" + "time" +) + +func TestSilentPeers(t *testing.T) { + cfg := DisabledConfig() + cfg.HeartbeatInterval = time.Second + local := [16]byte{1} + b, err := NewBroadcaster(cfg, local, 0, newMockHandler()) + if err != nil { + t.Fatal(err) + } + now := time.Now() + b.started = now.Add(-10 * time.Minute) + long := now.Add(-time.Minute) + + heard, newcomer, never, quiet := [16]byte{2}, [16]byte{3}, [16]byte{4}, [16]byte{5} + b.senders[heard] = &senderSeq{heard: now.Add(-time.Second)} + b.senders[quiet] = &senderSeq{heard: now.Add(-10 * time.Second)} + peers := []ExpectedPeer{ + {UUID: local, Name: "self", Mounted: long}, + {UUID: heard, Name: "heard", Mounted: long}, + {UUID: newcomer, Name: "newcomer", Mounted: now.Add(-2 * time.Second)}, + {UUID: never, Name: "never", Mounted: long}, + {UUID: quiet, Name: "quiet", Mounted: long}, + } + b.WatchPeers(func() ([]ExpectedPeer, error) { return peers, nil }) + + b.checkPeers(now) + names, after := b.SilentPeers() + if want := []string{"never", "quiet"}; !slices.Equal(names, want) { + t.Errorf("silent = %v, want %v", names, want) + } + if after != 3*time.Second { + t.Errorf("silent after %v, want 3 heartbeat intervals (3s)", after) + } + if got := b.Stats().SilentPeers; got != 2 { + t.Errorf("Stats().SilentPeers = %d, want 2", got) + } + + b.track(quiet, 1) + b.checkPeers(now) + if names, _ := b.SilentPeers(); !slices.Equal(names, []string{"never"}) { + t.Errorf("after hearing quiet: silent = %v, want [never]", names) + } + + b.WatchPeers(func() ([]ExpectedPeer, error) { return nil, errors.New("registry unreadable") }) + b.checkPeers(now) + if names, _ := b.SilentPeers(); !slices.Equal(names, []string{"never"}) { + t.Errorf("an unreadable registry changed the silent peers to %v", names) + } + + peers = slices.DeleteFunc(peers, func(p ExpectedPeer) bool { return p.UUID == never }) + b.WatchPeers(func() ([]ExpectedPeer, error) { return peers, nil }) + b.checkPeers(now) + if names, _ := b.SilentPeers(); len(names) != 0 { + t.Errorf("after never unmounted: silent = %v, want none", names) + } +} + +func TestSilentPeersWithoutWatch(t *testing.T) { + cfg := DisabledConfig() + cfg.HeartbeatInterval = time.Second + b, err := NewBroadcaster(cfg, [16]byte{1}, 0, newMockHandler()) + if err != nil { + t.Fatal(err) + } + b.checkPeers(time.Now()) + if names, _ := b.SilentPeers(); len(names) != 0 { + t.Errorf("silent = %v with nothing to watch", names) + } +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 8e83bacf..57d2ed39 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -156,6 +156,7 @@ func (m *Manager) Stats() Stats { stats.MulticastEnabled = bstats.Transport.MulticastEnabled stats.PeerCount = bstats.Transport.PeerCount stats.BroadcastSenders = bstats.TrackedSenders + stats.BroadcastSilentPeers = bstats.SilentPeers stats.BroadcastMaxPeers = bstats.MaxPeers } @@ -201,6 +202,7 @@ type Stats struct { MulticastEnabled bool PeerCount int BroadcastSenders int + BroadcastSilentPeers int BroadcastMaxPeers int } @@ -292,6 +294,23 @@ func (m *Manager) BroadcastProblem() string { return m.broadcaster.Problem() } +// WatchBroadcastPeers has the broadcaster report expected nodes it does not +// hear from. It does nothing with broadcast off. +func (m *Manager) WatchBroadcastPeers(expected func() ([]broadcast.ExpectedPeer, error)) { + if m.broadcaster != nil { + m.broadcaster.WatchPeers(expected) + } +} + +// SilentBroadcastPeers names the expected nodes this node has not heard on +// broadcast for the returned time. +func (m *Manager) SilentBroadcastPeers() ([]string, time.Duration) { + if m.broadcaster == nil { + return nil, 0 + } + return m.broadcaster.SilentPeers() +} + // BroadcastInvalidateInode invalidates an inode locally AND broadcasts // the invalidation to peer nodes. Call this after modifying an inode, and // cache the new inode after this call, since it drops the local entry. diff --git a/internal/stats/convert.go b/internal/stats/convert.go index 96058784..e695d0fc 100644 --- a/internal/stats/convert.go +++ b/internal/stats/convert.go @@ -36,7 +36,7 @@ func cacheResponse(c cache.Stats, l lock.Stats) *api.CacheStatsResponse { resp.ExtentCache.Capacity = c.ExtentCacheCapacity resp.NegativeCache.Capacity = c.NegativeCacheCapacity if c.BroadcastEnabled { - resp.Broadcast = &api.BroadcastTables{Peers: c.PeerCount, Senders: c.BroadcastSenders, MaxPeers: c.BroadcastMaxPeers} + resp.Broadcast = &api.BroadcastTables{Peers: c.PeerCount, Senders: c.BroadcastSenders, SilentPeers: c.BroadcastSilentPeers, MaxPeers: c.BroadcastMaxPeers} } return resp } diff --git a/tests/docker/docker-compose.test.yml b/tests/docker/docker-compose.test.yml index c94ae2c5..b5b48e7b 100644 --- a/tests/docker/docker-compose.test.yml +++ b/tests/docker/docker-compose.test.yml @@ -45,6 +45,8 @@ services: - MOUNT_POINT=/mnt/goclusterfs - FORMAT_ON_START=true - DEBUG=${DEBUG:-false} + # Short, so a node no longer heard on broadcast is reported within seconds + - BROADCAST_HEARTBEAT=${BROADCAST_HEARTBEAT:-2s} networks: cluster-net: ipv4_address: 192.168.168.11 @@ -77,6 +79,7 @@ services: - MOUNT_POINT=/mnt/goclusterfs - FORMAT_ON_START=false - DEBUG=${DEBUG:-false} + - BROADCAST_HEARTBEAT=${BROADCAST_HEARTBEAT:-2s} networks: cluster-net: ipv4_address: 192.168.168.12 @@ -109,6 +112,7 @@ services: - MOUNT_POINT=/mnt/goclusterfs - FORMAT_ON_START=false - DEBUG=${DEBUG:-false} + - BROADCAST_HEARTBEAT=${BROADCAST_HEARTBEAT:-2s} networks: cluster-net: ipv4_address: 192.168.168.13 @@ -141,6 +145,7 @@ services: - MOUNT_POINT=/mnt/goclusterfs - FORMAT_ON_START=false - DEBUG=${DEBUG:-false} + - BROADCAST_HEARTBEAT=${BROADCAST_HEARTBEAT:-2s} networks: cluster-net: ipv4_address: 192.168.168.14 diff --git a/tests/docker/scripts/entrypoint.sh b/tests/docker/scripts/entrypoint.sh index 4c85c65b..2ba47b28 100644 --- a/tests/docker/scripts/entrypoint.sh +++ b/tests/docker/scripts/entrypoint.sh @@ -72,6 +72,9 @@ log "Device $DEVICE_PATH is ready (size: $(stat -c%s "$DEVICE_PATH") bytes)" # Build mount command arguments MOUNT_ARGS="-device $DEVICE_PATH -mount $MOUNT_POINT" +if [ -n "$BROADCAST_HEARTBEAT" ]; then + MOUNT_ARGS="$MOUNT_ARGS -broadcast-heartbeat $BROADCAST_HEARTBEAT" +fi # Only format if FORMAT_ON_START=true AND we haven't already formatted # This prevents data loss on container restart diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 08a7e5dc..2969d2bd 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -2571,6 +2571,62 @@ test_staleness_flood_under_loss() { measure_staleness flood_under_loss flood } +# api_get prints the response to GET $2 from the API inside node $1. +api_get() { + exec_on_node "$1" bash -c "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET $2 HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && cat <&3" 2>/dev/null +} + +broadcast_peers_health() { + api_get "$1" /health | tr -d ' \n' | grep -o '"broadcast-peers":{"status":"[a-z]*"' | sed 's/.*"status":"//; s/"$//' +} + +silent_peers_metric() { + api_get "$1" /metrics | awk '/^goclusterfs_broadcast_silent_peers /{print $2}' +} + +# A node that hears no multicast must say so: on /health, on /metrics and in its log. +test_broadcast_silent_peer_reported() { + local reader="${NODE_ARRAY[1]}" other="${NODE_ARRAY[0]}" + local want=$((NODE_COUNT - 1)) status metric i failed=0 + local since + since=$(date +%s) + + apply_fault "$reader" iptables -I INPUT -d 239.192.74.75 -j DROP || return 1 + for i in $(seq 1 30); do + status=$(broadcast_peers_health "$reader") + metric=$(silent_peers_metric "$reader") + [[ "$status" == degraded && "$metric" == "$want" ]] && break + sleep 1 + done + log "After dropping multicast for ${i}s, $reader: broadcast-peers $status, silent peers $metric" + if [[ "$status" != degraded || "$metric" != "$want" ]]; then + log "ERROR: $reader did not report its $want peers as silent" + failed=1 + fi + if ! docker logs --since "$since" "$reader" 2>&1 | grep -q "no broadcast heard from node"; then + log "ERROR: $reader logged no warning about the silent peers" + failed=1 + fi + if [[ "$(broadcast_peers_health "$other")" != ok ]]; then + log "ERROR: $other, which still hears multicast, reports silent peers" + failed=1 + fi + + heal_network_faults + for i in $(seq 1 20); do + status=$(broadcast_peers_health "$reader") + metric=$(silent_peers_metric "$reader") + [[ "$status" == ok && "$metric" == 0 ]] && break + sleep 1 + done + log "${i}s after healing, $reader: broadcast-peers $status, silent peers $metric" + if [[ "$status" != ok || "$metric" != 0 ]]; then + log "ERROR: $reader still reports silent peers after multicast came back" + failed=1 + fi + return $failed +} + # Print the free 1MB blocks that statfs reports on node $1 statfs_free_blocks() { exec_on_node "$1" stat -f -c %f /mnt/goclusterfs @@ -3436,6 +3492,7 @@ run_test "Broadcast Invalidation" test_broadcast_invalidation run_test "Broadcast Directory Invalidation" test_broadcast_directory_invalidation run_test "Broadcast Multiple Updates" test_broadcast_multiple_updates run_test "Broadcast Bidirectional" test_broadcast_bidirectional +run_test "Broadcast Silent Peer Reported" test_broadcast_silent_peer_reported run_test "Staleness Under Packet Loss" test_staleness_packet_loss run_test "Staleness Under Duplication and Reordering" test_staleness_duplication_and_reordering run_test "Staleness Under Delay" test_staleness_delay diff --git a/tests/integration/broadcast_test.go b/tests/integration/broadcast_test.go index 9e2515b4..70bd8c87 100644 --- a/tests/integration/broadcast_test.go +++ b/tests/integration/broadcast_test.go @@ -15,6 +15,8 @@ import ( "strings" "testing" "time" + + "github.com/supporttools/GoClusterFS/internal/api" ) // Well above the 5s inode cache TTL; tests log propagation time instead of asserting a bound. @@ -332,3 +334,58 @@ func TestMountFailsOnTakenBroadcastPortBeforeFormat(t *testing.T) { t.Error("mount wrote the superblock before failing on the broadcast port") } } + +// TestBroadcastReportsSilentPeer mounts a node that sends no broadcasts next +// to one that does. The broadcasting node must report it on /health and +// /metrics: it would otherwise see that node's changes only after -cache-ttl +// without anything saying so. +func TestBroadcastReportsSilentPeer(t *testing.T) { + cluster := NewMultiNodeCluster(t, 2, 50) + addr := freeLoopbackAddr(t) + cluster.Node(0).ExtraArgs = []string{"-api-addr", addr, "-broadcast-heartbeat", "500ms"} + cluster.Node(1).ExtraArgs = []string{"-broadcast=false"} + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("FormatAndStartFirst failed: %v", err) + } + base := "http://" + addr + waitForAPI(t, base) + + var health api.HealthResponse + getAPI(t, base+"/health", &health) + if c := health.Checks["broadcast-peers"]; c.Status != api.HealthStatusOK { + t.Fatalf("broadcast-peers before the second node mounted: %+v", c) + } + + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("StartNode(1) failed: %v", err) + } + elapsed, ok := waitFor(10*time.Second, func() bool { + getAPI(t, base+"/health", &health) + return health.Checks["broadcast-peers"].Status == api.HealthStatusDegraded + }) + if !ok { + t.Fatalf("node 0 never reported node 1 as silent: %+v", health.Checks["broadcast-peers"]) + } + t.Logf("reported %v after node 1 mounted: %s", elapsed.Round(100*time.Millisecond), health.Checks["broadcast-peers"].Message) + + if _, ok := waitFor(5*time.Second, func() bool { + v, ok := metricValue(getMetrics(t, base), "goclusterfs_broadcast_silent_peers") + return ok && v == 1 + }); !ok { + t.Errorf("goclusterfs_broadcast_silent_peers never reached 1") + } +} + +func getMetrics(t *testing.T, base string) string { + t.Helper() + resp, err := apiClient.Get(base + "/metrics") + if err != nil { + return "" + } + defer mustClose(t, resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return "" + } + return string(body) +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 93bfaae2..1d6fd354 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2360,6 +2360,22 @@ scenarios: environments: go: internal/cache/broadcast/broadcaster_test.go::TestTrackedSendersBounded + - id: broadcast-silent-peers + name: Silent broadcast peers are reported + description: A node the registry shows mounted but not heard for 3 heartbeat intervals counts as silent until it is heard again or unmounts; this node is never listed + priority: go_only + environments: + go: internal/cache/broadcast/silence_test.go::TestSilentPeers + + - id: broadcast-silent-peer-reported + name: A node reports peers it hears no broadcasts from + description: A node next to one that sends no broadcasts (Go) or that hears no multicast (Docker) reports its peers on /health and /metrics, and recovers when they are heard again + priority: high + environments: + go: broadcast_test.go::TestBroadcastReportsSilentPeer + docker: run-tests.sh::test_broadcast_silent_peer_reported + vm: TODO + - id: broadcast-loss-drops-caches name: Broadcast loss drops caches description: A gap in a peer's sequence numbers clears the inode and extent caches once; a late message does not clear them again From a370ff16af4f6e8ff4752fa202cdd42651998b63 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 13:45:03 -0600 Subject: [PATCH 2/2] fix(broadcast): time a peer's silence from when this node first listed it The grace period used the peer's MountTime from the registry, written by the peer's clock, against this node's clock, so a peer with a clock ahead went unreported for up to the skew. Silence now counts from when this node first saw the peer in the registry, by its own clock, as #151 did for liveness. Co-Authored-By: Claude Opus 5.5 --- cmd/goclusterfs-mount/peers.go | 6 ++-- internal/cache/broadcast/receiver.go | 9 +++--- internal/cache/broadcast/silence.go | 35 ++++++++++++++---------- internal/cache/broadcast/silence_test.go | 35 ++++++++++++++---------- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/cmd/goclusterfs-mount/peers.go b/cmd/goclusterfs-mount/peers.go index a870b812..f4d585b8 100644 --- a/cmd/goclusterfs-mount/peers.go +++ b/cmd/goclusterfs-mount/peers.go @@ -3,7 +3,6 @@ package main import ( "fmt" "strings" - "time" "github.com/supporttools/GoClusterFS/internal/cache/broadcast" "github.com/supporttools/GoClusterFS/internal/node" @@ -21,9 +20,8 @@ func expectedPeers(reg *node.Registry) func() ([]broadcast.ExpectedPeer, error) for _, s := range slots { host := strings.TrimRight(string(s.Hostname[:]), "\x00") peers = append(peers, broadcast.ExpectedPeer{ - UUID: s.UUID, - Name: fmt.Sprintf("%s (%x)", host, s.UUID[:4]), - Mounted: time.Unix(s.MountTime, 0), + UUID: s.UUID, + Name: fmt.Sprintf("%s (%x)", host, s.UUID[:4]), }) } return peers, nil diff --git a/internal/cache/broadcast/receiver.go b/internal/cache/broadcast/receiver.go index 3560224d..56a955fc 100644 --- a/internal/cache/broadcast/receiver.go +++ b/internal/cache/broadcast/receiver.go @@ -38,10 +38,11 @@ type Broadcaster struct { sendersMu sync.Mutex senders map[[16]byte]*senderSeq - started time.Time - expected atomic.Pointer[func() ([]ExpectedPeer, error)] - silentMu sync.Mutex - silent map[[16]byte]ExpectedPeer + started time.Time + expected atomic.Pointer[func() ([]ExpectedPeer, error)] + silentMu sync.Mutex + silent map[[16]byte]ExpectedPeer + firstListed map[[16]byte]time.Time lossLogMu sync.Mutex lossLoggedAt time.Time diff --git a/internal/cache/broadcast/silence.go b/internal/cache/broadcast/silence.go index 8b36c313..3e5f6c97 100644 --- a/internal/cache/broadcast/silence.go +++ b/internal/cache/broadcast/silence.go @@ -12,16 +12,15 @@ const silentAfter = 3 // ExpectedPeer is a node that should be sending this node broadcasts. type ExpectedPeer struct { - UUID [16]byte - Name string - Mounted time.Time + UUID [16]byte + Name string } // WatchPeers has the broadcaster check, every heartbeat interval, that it // hears from each node that expected lists. A node silent for silentAfter -// intervals since it mounted or this node started is reported: dropped -// multicast is otherwise invisible, because a node that hears nothing from a -// peer finds no gap in its sequence numbers either. +// intervals since this node first saw it listed, by this node's clock, is +// reported: dropped multicast is otherwise invisible, because a node that +// hears nothing from a peer finds no gap in its sequence numbers either. func (b *Broadcaster) WatchPeers(expected func() ([]ExpectedPeer, error)) { b.expected.Store(&expected) } @@ -70,17 +69,23 @@ func (b *Broadcaster) checkPeers(now time.Time) { } quiet := b.silenceThreshold() - mounted := map[[16]byte]bool{} + b.silentMu.Lock() + defer b.silentMu.Unlock() + listed := make(map[[16]byte]time.Time, len(peers)) silent := map[[16]byte]ExpectedPeer{} b.sendersMu.Lock() for _, p := range peers { if p.UUID == b.localUUID { continue } - mounted[p.UUID] = true + first, ok := b.firstListed[p.UUID] + if !ok { + first = now + } + listed[p.UUID] = first last := b.started - if p.Mounted.After(last) { - last = p.Mounted + if first.After(last) { + last = first } if s, ok := b.senders[p.UUID]; ok && s.heard.After(last) { last = s.heard @@ -91,8 +96,6 @@ func (b *Broadcaster) checkPeers(now time.Time) { } b.sendersMu.Unlock() - b.silentMu.Lock() - defer b.silentMu.Unlock() for id, p := range silent { if _, ok := b.silent[id]; !ok { log.Printf("[broadcast] WARN: no broadcast heard from node %s for over %v, though the registry shows it mounted; this node sees its changes only after -cache-ttl. Multicast may not reach this node, or that node runs with -broadcast=false or another -mcast-group or -mcast-port", @@ -100,9 +103,11 @@ func (b *Broadcaster) checkPeers(now time.Time) { } } for id, p := range b.silent { - if _, ok := silent[id]; !ok && mounted[id] { - log.Printf("[broadcast] heard from node %s again", p.Name) + if _, ok := silent[id]; !ok { + if _, still := listed[id]; still { + log.Printf("[broadcast] heard from node %s again", p.Name) + } } } - b.silent = silent + b.silent, b.firstListed = silent, listed } diff --git a/internal/cache/broadcast/silence_test.go b/internal/cache/broadcast/silence_test.go index 73e8f0b5..aeaaf038 100644 --- a/internal/cache/broadcast/silence_test.go +++ b/internal/cache/broadcast/silence_test.go @@ -7,6 +7,8 @@ import ( "time" ) +// TestSilentPeers counts silence from when this node first saw each peer +// listed, by its own clock, so a peer's clock cannot delay the report. func TestSilentPeers(t *testing.T) { cfg := DisabledConfig() cfg.HeartbeatInterval = time.Second @@ -17,21 +19,21 @@ func TestSilentPeers(t *testing.T) { } now := time.Now() b.started = now.Add(-10 * time.Minute) - long := now.Add(-time.Minute) - heard, newcomer, never, quiet := [16]byte{2}, [16]byte{3}, [16]byte{4}, [16]byte{5} - b.senders[heard] = &senderSeq{heard: now.Add(-time.Second)} - b.senders[quiet] = &senderSeq{heard: now.Add(-10 * time.Second)} - peers := []ExpectedPeer{ - {UUID: local, Name: "self", Mounted: long}, - {UUID: heard, Name: "heard", Mounted: long}, - {UUID: newcomer, Name: "newcomer", Mounted: now.Add(-2 * time.Second)}, - {UUID: never, Name: "never", Mounted: long}, - {UUID: quiet, Name: "quiet", Mounted: long}, - } + heard, never, quiet, newcomer := [16]byte{2}, [16]byte{3}, [16]byte{4}, [16]byte{5} + peers := []ExpectedPeer{{local, "self"}, {heard, "heard"}, {never, "never"}, {quiet, "quiet"}} b.WatchPeers(func() ([]ExpectedPeer, error) { return peers, nil }) b.checkPeers(now) + if names, _ := b.SilentPeers(); len(names) != 0 { + t.Errorf("silent = %v at the first look, before any grace period passed", names) + } + + later := now.Add(4 * time.Second) + peers = append(peers, ExpectedPeer{newcomer, "newcomer"}) + b.senders[heard] = &senderSeq{heard: later.Add(-time.Second)} + b.senders[quiet] = &senderSeq{heard: now.Add(-10 * time.Second)} + b.checkPeers(later) names, after := b.SilentPeers() if want := []string{"never", "quiet"}; !slices.Equal(names, want) { t.Errorf("silent = %v, want %v", names, want) @@ -43,24 +45,27 @@ func TestSilentPeers(t *testing.T) { t.Errorf("Stats().SilentPeers = %d, want 2", got) } - b.track(quiet, 1) - b.checkPeers(now) + b.senders[quiet].heard = later + b.checkPeers(later) if names, _ := b.SilentPeers(); !slices.Equal(names, []string{"never"}) { t.Errorf("after hearing quiet: silent = %v, want [never]", names) } b.WatchPeers(func() ([]ExpectedPeer, error) { return nil, errors.New("registry unreadable") }) - b.checkPeers(now) + b.checkPeers(later) if names, _ := b.SilentPeers(); !slices.Equal(names, []string{"never"}) { t.Errorf("an unreadable registry changed the silent peers to %v", names) } peers = slices.DeleteFunc(peers, func(p ExpectedPeer) bool { return p.UUID == never }) b.WatchPeers(func() ([]ExpectedPeer, error) { return peers, nil }) - b.checkPeers(now) + b.checkPeers(later) if names, _ := b.SilentPeers(); len(names) != 0 { t.Errorf("after never unmounted: silent = %v, want none", names) } + if _, ok := b.firstListed[never]; ok { + t.Error("an unmounted peer is still tracked") + } } func TestSilentPeersWithoutWatch(t *testing.T) {