From 825eadd19eb4f7e33a3442cdc4a81f66cc5d671a Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:06:33 +0800 Subject: [PATCH] fix(timeline): warn once per export failure streak and report it in health (#2834) a dead collector warned on every tick: at the default 5s interval that is 17,280 warnings a day, and the operator cannot tell one outage from many. the cursor now counts consecutive failures, warns when the streak opens and when it recovers, and timeline/health carries failures/lastOkAt/lastError. --- ...34-timeline-export-failure-streak.fixed.md | 1 + docs/subsystems/timeline.md | 8 +- plugins/timeline/src/health.js | 16 +++- plugins/timeline/src/otel.js | 20 ++++- plugins/timeline/src/timeline.js | 4 +- .../test/export-failure-streak.test.js | 80 +++++++++++++++++++ 6 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 changelog.d/2834-timeline-export-failure-streak.fixed.md create mode 100644 plugins/timeline/test/export-failure-streak.test.js diff --git a/changelog.d/2834-timeline-export-failure-streak.fixed.md b/changelog.d/2834-timeline-export-failure-streak.fixed.md new file mode 100644 index 000000000..41e123fc0 --- /dev/null +++ b/changelog.d/2834-timeline-export-failure-streak.fixed.md @@ -0,0 +1 @@ +A timeline export that keeps failing now warns once when the streak opens and once when it recovers, instead of once per tick, and `timeline/health` reports the exporter state (`failures`, `lastOkAt`, `lastError`) so a broken collector is visible without reading the log. diff --git a/docs/subsystems/timeline.md b/docs/subsystems/timeline.md index cfbba268d..30b0e341e 100644 --- a/docs/subsystems/timeline.md +++ b/docs/subsystems/timeline.md @@ -29,7 +29,7 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model | `plugins/timeline/src/recorder.js` | Feed events to compact trace records; rebuilds every field from a strict alphabet. | | `plugins/timeline/src/store.js` | The trace: single writer, write-tmp + fsync + rename, capped segments. | | `plugins/timeline/src/window.js` | Pure read model: records to lanes, turns, tools, arrows, edges, utilization, the token spend per Bot (#357), per-Bot outcomes and recent failures (#471), the open requests waiting longest (#469) and the tools each Bot used (#359). | -| `plugins/timeline/src/health.js` | Team health (#357): pure `buildHealth(window)` turning the window into per-Bot signals and flags. | +| `plugins/timeline/src/health.js` | Team health (#357): pure `buildHealth(window, exportState)` turning the window into per-Bot signals and flags. The optional second argument is the exporter's streak (`{ failures, lastOkAt, lastError }`) and adds `export` (P-699); `null` reads `export: { enabled: false }`, and without the argument the answer carries no `export` key. | | `plugins/timeline/src/digest.js` | Team run report (#582): pure `buildDigest(window)` turning the window into short Markdown, with the count and duration formatting (`formatCount`, `formatMs`). A duration rounds once and carries into the next unit (#761), so the text says `1m` and `2h00m`, never `60s`, `60m` or `1h60m`. A count carries the same way (#1779): 999,950 reads `1.0M`, never `1000.0k`. `busy` is the share of the window busy, not a count of turns (invariant 5): a Bot whose turn began before the window reports its busy share instead of reading as a turn-less Bot, the same answer `timeline/health` gives over that window. | | `plugins/timeline/src/alerts.js` | Durable alerts (#584): pure `createAlertTracker()` folding a window across reads, so a condition opens one alert and resolves when it clears. | | `plugins/timeline/client.js` | Browser entry (`@flybot/timeline`): the 2 s poller exposed as `ctx.flybot.timeline`, the geometry functions (`layoutSwimlanes`, `layoutMap`, `waitRows`, `utilizationRows`) and the panel (swimlanes, who waits on whom, utilization, and **Copy report**), opened from its `flybot.sidebar.section` row, the only entry (not in the + menu, #269). | @@ -41,7 +41,7 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model - `plugins/timeline/index.js` — loaded as `@flybot/timeline`. - Endpoint `timeline/window` (wire name `flybot/timeline/window`), registered `{ read: true }` so a read-only host still answers. Body `{ windowMs?, to? }`; window 30 s to 24 h, default 15 min. - Endpoint `timeline/export` (#357, read-only): the current trace mapped to one OTLP/HTTP JSON payload (`resourceSpans`), the same mapping the timer exporter uses. Ids only unless the body sets `names: true` (#483); `serviceName` overrides the default. The panel or an automation can pull it on demand instead of waiting for the next flush. -- Endpoint `timeline/health` (#357, read-only): the same window reduced to per-Bot signals and flags (`errors`, `busy`, `idle`, `waiting`) and a team `ok`; the action view over the numbers the panel draws. Body `{ windowMs?, to? }` like `timeline/window`. +- Endpoint `timeline/health` (#357, read-only): the same window reduced to per-Bot signals and flags (`errors`, `busy`, `idle`, `waiting`) and a team `ok`; the action view over the numbers the panel draws. Body `{ windowMs?, to? }` like `timeline/window`. With an exporter configured the answer also carries `export: { enabled: true, failing, failures, lastOkAt, lastError }` (P-699), the same streak the exporter warns about; without one it reads `export: { enabled: false }`. `failures` counts consecutive failed sends, `lastOkAt` is when a send last succeeded (ms, `null` if none), and `lastError` is the message of the newest failure (`null` once a send succeeds). - Endpoint `timeline/alerts` (#584, read-only): the same window folded across reads into durable alerts — `errors` (a Bot fails most of its turns), `waiting` (its request is unanswered) and `stuck` (a handoff waited past the threshold) — answered as `{ open, resolved, changed, stale }`. Body `{ windowMs?, to? }` like `timeline/window`. A condition opens ONE alert with its `since` and resolves when it clears; a resolved alert stays readable for a minute after it cleared. - Endpoint `timeline/digest` (#582, read-only): the same window as a short Markdown run report a person can paste into a chat or a standup — per Bot its turns and failures, busy share, tokens (in/out/cached), top tools and unanswered requests, then the team rollup (turns, failures, tokens, calls) and the longest still-open wait. Body `{ windowMs?, to?, title?, timeZone? }`. It reads only what the window carries: `usage`, `toolUse` and `stuck` when present, and nothing is invented when they are absent (`buildDigest`/`formatCount`/`formatMs` in `src/digest.js`, pure). Window hours are the reader's own clock, the same hours the panel header shows, so what a person copies matches what they see above the button. The panel sends its own browser zone (`timeZone`) with the request, and the host formats the window in it — including the day suffix — falling back to its own clock when the zone is not one it can use (#2619). The panel's **Copy report** button asks for it and copies the Markdown; a page that refuses the clipboard says so instead of reporting a copy. @@ -57,12 +57,12 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model 8. The panel draws text only: every string is a React text child or an SVG text node, never markup, and is cleaned again in the browser (`safe`). It polls only while open (and refreshes at once when the tab or window comes back, #259, #271), shows the last good data under an error, and says in a note when the trace is shorter than the window, truncated, memory-only or looking back. Its one-line footnote repeats the honesty rules of invariant 5 (delivered is not read, busy is not idle, model time is not a measured latency). The dialog shrinks to its content and hangs from the top (switching tabs moves nothing above the body), the **Live** control is always shown (pressed while following the latest activity, a call to action when looking back), lanes name what they are under the name (`main`, `group`, `friend`, `removed`) with no icon prefix, and the map puts each name beside or under its node, clipped with the full name in a tooltip (#249). A turn bar's tooltip ends with the model the turn ran on (its model id, `· deepseek-v4-pro`); a running turn, or one recorded before #366, names none (#366). A friend's Bot is named as the sidebar names it (`UiA1415's Chief`, `Helper · UiB1415's Bot`, from the feed's `remote.name` / party `name`), so two Bots called Helper stay apart; a wait label that would run past the plot is right-aligned, and one with no room is left to the arrow's tooltip (#255). The window's range is always in view under the header (`Oct 18 · 13:20:05 – 13:35:05 (15 min)`, dates at both ends across midnight, and whether it is live), and the footnote is two short items, not one line (#279). The wait table below the map prints `–` wherever a number does not exist yet: an edge nobody answered has no median, slowest or total wait, so all three read `–` rather than a measured `0 ms` beside a `Not yet 1`. 9. Opt-in (#281): `apply` without `config.enabled === true` subscribes to nothing, writes nothing and registers no endpoint; the browser entry draws no sidebar row unless `config.sidebarSection === true` and its poller only runs while a panel is open. A test asserts the host does nothing by default. Docs and the install doctor's default commands do not present it as part of a standard install. 10. The endpoint reads only. It does not start, stop, send or change anything. -11. OTLP export (#483) is opt-in and off by default: without `config.exporter` nothing leaves the host. With `exporter: "otlp"` the plugin POSTs OTLP/HTTP JSON to `otlpUrl`; with `"json"` it appends `dir/export.jsonl` (capped at 8 MiB, then it stops and warns). `otlpUrl` must be `https`, or `http` on localhost only (the request may carry an auth header). `exportMs` is the export interval: capped at 2147483647 ms (Node's `setInterval` ceiling, about 24.8 days), a larger value is clamped to that ceiling with one warning naming the interval in force, because Node would otherwise shorten the delay to 1 ms and run the exporter on every tick. Only CLOSED spans are exported: a turn needs its `te` and a tool call its `ke`, so an open turn is carried between flushes and exported when it ends, never as a finished success. A turn or tool call whose pair of timestamps cannot both be read as finite numbers — an old or hand-edited record the trace still keeps — builds no span, the way the window view already skips it, instead of failing the whole export (#2442). The exporter reads only the same compact records the panel does (invariant 2 still holds: no message text, arguments or output), maps turns to `invoke_agent` spans and tool calls to `execute_tool` spans under them, pairs a tool call by its turn and call id (`tid`/`id`, the pair the window folds by) so two turns that both call a tool `call_0` still export two spans, and carries every id through the same deterministic hash. Bot and room NAMES are sent only with `exportNames: true`; by default a span carries ids only, so a friend's or another home's name never leaves the host (a name comes from `trace.names()`, which includes friends' Bots and face-room names). The feed has no model name and no token counts, so those GenAI attributes are absent, not guessed. A failed send keeps the cursor, so the next flush retries the same closed spans (tested). The POST refuses a redirect (`redirect: 'error'`), as every other credentialed request in this repository does: following a collector's `307` replays `otlpHeaders` and the span payload at the `Location` origin the operator never named, and the export counts as delivered. A redirect is an export failure instead — warned, not counted, retried on the next flush (#2687). The cursor re-reads records at its last timestamp (`>=`), because a `ke` can share the millisecond of its `ks` (or a `te` the time of the last record); a span already sent is deduplicated by id, so a same-millisecond record is neither skipped nor sent twice. +11. OTLP export (#483) is opt-in and off by default: without `config.exporter` nothing leaves the host. With `exporter: "otlp"` the plugin POSTs OTLP/HTTP JSON to `otlpUrl`; with `"json"` it appends `dir/export.jsonl` (capped at 8 MiB, then it stops and warns). `otlpUrl` must be `https`, or `http` on localhost only (the request may carry an auth header). `exportMs` is the export interval: capped at 2147483647 ms (Node's `setInterval` ceiling, about 24.8 days), a larger value is clamped to that ceiling with one warning naming the interval in force, because Node would otherwise shorten the delay to 1 ms and run the exporter on every tick. Only CLOSED spans are exported: a turn needs its `te` and a tool call its `ke`, so an open turn is carried between flushes and exported when it ends, never as a finished success. A turn or tool call whose pair of timestamps cannot both be read as finite numbers — an old or hand-edited record the trace still keeps — builds no span, the way the window view already skips it, instead of failing the whole export (#2442). The exporter reads only the same compact records the panel does (invariant 2 still holds: no message text, arguments or output), maps turns to `invoke_agent` spans and tool calls to `execute_tool` spans under them, pairs a tool call by its turn and call id (`tid`/`id`, the pair the window folds by) so two turns that both call a tool `call_0` still export two spans, and carries every id through the same deterministic hash. Bot and room NAMES are sent only with `exportNames: true`; by default a span carries ids only, so a friend's or another home's name never leaves the host (a name comes from `trace.names()`, which includes friends' Bots and face-room names). The feed has no model name and no token counts, so those GenAI attributes are absent, not guessed. A failed send keeps the cursor, so the next flush retries the same closed spans (tested). A failing send is warned once when its streak opens and once when it recovers, never once per attempt: a dead collector at the default 5 s interval used to write 17,280 warnings a day (P-699), and `timeline/health` carries the `failures` / `lastOkAt` / `lastError` streak the operator can read instead. The POST refuses a redirect (`redirect: 'error'`), as every other credentialed request in this repository does: following a collector's `307` replays `otlpHeaders` and the span payload at the `Location` origin the operator never named, and the export counts as delivered. A redirect is an export failure instead — warned, not counted, retried on the next flush (#2687). The cursor re-reads records at its last timestamp (`>=`), because a `ke` can share the millisecond of its `ks` (or a `te` the time of the last record); a span already sent is deduplicated by id, so a same-millisecond record is neither skipped nor sent twice. 12. Durable alerts (#584) keep their state in the plugin process, not in the trace: the tracker opens one alert per condition (`errors:`, `waiting:`, `stuck:`) and resolves it when the condition clears, keeping the resolution readable for a minute so a reader polling slower than the transition still sees that it cleared. `changed` is true only when something opened or resolved; a detail that moves while the alert keeps holding is not a transition. `stuck` uses the same 20 open requests the window already carries and a 15 min default threshold (`stuckMs`), which the issue left open. A window that ends before the last one folded is answered `stale` with the state untouched, so a look-back never resolves a live alert. The state dies with the process: nothing is written, and a restart starts with no alerts rather than replaying old ones. ## Tests -`plugins/timeline/test/`: panel (geometry, a small hook renderer for open/close, tabs and poll hold, hostile-name negatives), sanitizers, trace store (rotation, retention, lock, crash-safe replace), recorder (negative tests), read model, OTLP export (`otel.test.js`: record→span mapping, payload shape, the cursor and its retry; `otlp-redirect.test.js`: two local servers, a `307` collector whose `Location` origin receives nothing and counts no export, and a `200` still delivered; `export-interval-ceiling.test.js`: the delay actually handed to `setInterval`, the clamp above Node's 32-bit timer ceiling and its one warning), durable alerts (`alerts.test.js`: one alert per condition, resolution and its grace window, the stuck threshold, the stale look-back, the endpoint and its removal on dispose), the run report (`digest.test.js`: the pure report, and `timeline/digest` through a real host, read-only and gone after dispose), and a host test on the real `ctx.flybot` with a mini agent loop on `core/dev/mock-llm.mjs`. Run `npm test` in `plugins/timeline` (CI job `plugins (timeline)`). +`plugins/timeline/test/`: panel (geometry, a small hook renderer for open/close, tabs and poll hold, hostile-name negatives), sanitizers, trace store (rotation, retention, lock, crash-safe replace), recorder (negative tests), read model, OTLP export (`otel.test.js`: record→span mapping, payload shape, the cursor and its retry; `otlp-redirect.test.js`: two local servers, a `307` collector whose `Location` origin receives nothing and counts no export, and a `200` still delivered; `export-interval-ceiling.test.js`: the delay actually handed to `setInterval`, the clamp above Node's 32-bit timer ceiling and its one warning; `export-failure-streak.test.js`: one warning per failure streak and one on recovery, the `timeline/health` export section, and a 100 ms interval against a dead collector warning once in 560 ms), durable alerts (`alerts.test.js`: one alert per condition, resolution and its grace window, the stuck threshold, the stale look-back, the endpoint and its removal on dispose), the run report (`digest.test.js`: the pure report, and `timeline/digest` through a real host, read-only and gone after dispose), and a host test on the real `ctx.flybot` with a mini agent loop on `core/dev/mock-llm.mjs`. Run `npm test` in `plugins/timeline` (CI job `plugins (timeline)`). ## Common mistakes diff --git a/plugins/timeline/src/health.js b/plugins/timeline/src/health.js index cd6d271da..d5dd09850 100644 --- a/plugins/timeline/src/health.js +++ b/plugins/timeline/src/health.js @@ -15,8 +15,14 @@ const round = value => Math.round(value * 1000) / 1000 * whether it is running now, how many of its requests are unanswered, and flags * (`errors` / `busy` / `idle` / `waiting`). `ok` is false when any Bot has an `errors` or * `waiting` flag; `unhealthy` names them. + * + * `exportState` (P-699) is the exporter's streak, `{ failures, lastOkAt, lastError }`; the answer + * then carries `export: { enabled: true, failing, failures, lastOkAt, lastError }`, so a broken + * collector is visible where the operator looks rather than only in a repeated warning. `null` + * means no exporter is configured and reads `export: { enabled: false }`. Left out, the answer has + * no `export` key at all, so a caller that reads only the team signals is unchanged. */ -export function buildHealth(window = {}) { +export function buildHealth(window = {}, exportState = undefined) { const lanes = new Map((window.lanes ?? []).map(lane => [lane.id, lane])) // Unanswered requests per sender Bot: where a Bot's work is waiting on someone else. const waitingOn = new Map() @@ -50,5 +56,11 @@ export function buildHealth(window = {}) { } }) const unhealthy = bots.filter(bot => bot.flags.some(flag => flag.code === 'errors' || flag.code === 'waiting')).map(bot => bot.id) - return { version: 1, now: window.now ?? null, windowMs: window.windowMs ?? null, live: window.live === true, bots, unhealthy, ok: unhealthy.length === 0 } + const health = { version: 1, now: window.now ?? null, windowMs: window.windowMs ?? null, live: window.live === true, bots, unhealthy, ok: unhealthy.length === 0 } + if (exportState !== undefined) { + health.export = exportState === null + ? { enabled: false } + : { enabled: true, failing: (exportState.failures ?? 0) > 0, failures: exportState.failures ?? 0, lastOkAt: exportState.lastOkAt ?? null, lastError: exportState.lastError ?? null } + } + return health } diff --git a/plugins/timeline/src/otel.js b/plugins/timeline/src/otel.js index 61c6bc0df..e45dc10ea 100644 --- a/plugins/timeline/src/otel.js +++ b/plugins/timeline/src/otel.js @@ -173,10 +173,16 @@ export function otlpPayload(spans, { serviceName = 'flybot', serviceVersion = SC * is never sent twice. A send failure is reported through `warn` and the cursor stays put, so the * next push retries. `exportNames` off (the default) sends ids only, never Bot or room names. */ -export function createExporter({ send, serviceName = 'flybot', serviceVersion = SCOPE_VERSION, warn = () => {}, minRecords = 1, exportNames = false } = {}) { +export function createExporter({ send, serviceName = 'flybot', serviceVersion = SCOPE_VERSION, warn = () => {}, minRecords = 1, exportNames = false, now = Date.now } = {}) { let lastT = -1 let carried = [] const emitted = new Set() + // P-699: a dead collector used to warn once per tick, so a 5 s interval wrote 17,280 lines a day + // and the operator could not tell one outage from many. The streak is counted here: one warning + // when it opens, one when it closes, and the running count is read through `stats()`. + let failures = 0 + let lastOkAt = null + let lastError = null const remember = (spanId) => { emitted.add(spanId) if (emitted.size > 5000) emitted.delete(emitted.values().next().value) @@ -206,11 +212,21 @@ export function createExporter({ send, serviceName = 'flybot', serviceVersion = for (const span of spans) remember(span.spanId) carried = nextCarried lastT = nextT + if (failures > 0) { + warn(`flybot-timeline: OTLP export recovered after ${failures} failed attempt(s) (last error: ${lastError})`) + failures = 0 + lastError = null + } + lastOkAt = now() return spans.length } catch (error) { - warn('flybot-timeline: OTLP export failed: %s', error?.message ?? error) + const message = error?.message ?? String(error) + failures += 1 + lastError = message + if (failures === 1) warn(`flybot-timeline: OTLP export failed: ${message}; later failures stay quiet until it recovers (timeline/health carries the count)`) return 0 } }, + stats() { return { failures, lastOkAt, lastError } }, } } diff --git a/plugins/timeline/src/timeline.js b/plugins/timeline/src/timeline.js index 2f3e8ba53..3ed85cf35 100644 --- a/plugins/timeline/src/timeline.js +++ b/plugins/timeline/src/timeline.js @@ -38,8 +38,10 @@ export function createTimeline(adapter, { dir, persist = true, now = Date.now, e // OTLP export is opt-in: without an `exporter` (send + intervalMs) nothing leaves the host. let exportTimer let exportNow + let exportCursor if (exporter && typeof exporter.send === 'function') { const cursor = createExporter({ send: exporter.send, serviceName: exporter.serviceName, serviceVersion: exporter.serviceVersion, exportNames: exporter.exportNames === true, warn: adapter.warn }) + exportCursor = cursor exportNow = async () => { try { return await cursor.push(trace.records(), trace.names()) } catch (error) { adapter.warn('flybot-timeline: export failed: %s', error?.message ?? error); return 0 } } const configuredIntervalMs = Number.isFinite(exporter.intervalMs) && exporter.intervalMs > 0 ? exporter.intervalMs : undefined const intervalMs = configuredIntervalMs === undefined ? DEFAULT_EXPORT_MS : Math.min(configuredIntervalMs, MAX_EXPORT_INTERVAL_MS) @@ -70,7 +72,7 @@ export function createTimeline(adapter, { dir, persist = true, now = Date.now, e const exportWindow = (body = {}) => otlpPayload(spansFrom(trace.records(), { names: body?.names === true ? trace.names() : {} }), { serviceName: body?.serviceName }) const offExport = adapter.handle(EXPORT_ENDPOINT, exportWindow) // Team health (#357): the same window, reduced to per-Bot signals and flags. Read-only. - const health = (body = {}) => buildHealth(window(body)) + const health = (body = {}) => buildHealth(window(body), exportCursor ? exportCursor.stats() : null) const offHealth = adapter.handle(HEALTH_ENDPOINT, health) // Durable alerts (#584): the same window, folded across reads so a condition opens once and // resolves when it clears. Read-only and in memory; nothing new reaches the trace. diff --git a/plugins/timeline/test/export-failure-streak.test.js b/plugins/timeline/test/export-failure-streak.test.js new file mode 100644 index 000000000..3a5cd07c6 --- /dev/null +++ b/plugins/timeline/test/export-failure-streak.test.js @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { createExporter } from '../src/otel.js' +import { createTimeline, HEALTH_ENDPOINT } from '../src/timeline.js' +import { buildHealth } from '../src/health.js' +import { mockAdapter } from './mock-adapter.js' + +// One closed turn at `t`: enough for the cursor to build a span and call `send`. Each push uses a +// later turn, because the cursor keeps every span it sent and would skip a repeated one. +const records = (t = 1_000) => [ + { r: 'ts', t, id: `a#${t}`, pl: 'chat', bot: 'a' }, + { r: 'te', t: t + 600, id: `a#${t}`, o: 'completed' }, +] + +const collectorDown = async () => { throw new Error('HTTP 503') } + +test('#P699 a failing export warns once per streak, not once per attempt', async () => { + const warns = [] + const exporter = createExporter({ send: collectorDown, warn: (...args) => warns.push(args.join(' ')), now: () => 5_000 }) + for (let attempt = 0; attempt < 5; attempt++) await exporter.push(records()) + assert.equal(warns.length, 1, `a failing export warned ${warns.length} time(s) in 5 attempts, so the operator cannot tell one outage from five separate faults`) + assert.match(warns[0], /export failed: HTTP 503/) + assert.deepEqual(exporter.stats(), { failures: 5, lastOkAt: null, lastError: 'HTTP 503' }) +}) + +test('#P699 a recovery warns once, and the next streak warns again', async () => { + const warns = [] + let down = true + const exporter = createExporter({ send: async () => { if (down) throw new Error('HTTP 503') }, warn: (...args) => warns.push(args.join(' ')), now: () => 5_000 }) + await exporter.push(records(1_000)) + await exporter.push(records(2_000)) + down = false + await exporter.push(records(3_000)) + down = true + await exporter.push(records(4_000)) + assert.equal(warns.length, 3, `a failure streak, its recovery and a second streak should read as three warnings; warnings were ${JSON.stringify(warns)}`) + assert.match(warns[1], /recovered after 2 failed attempt\(s\) \(last error: HTTP 503\)/, 'the recovery must be one line naming the streak it ended') + assert.match(warns[0], /export failed/) + assert.match(warns[2], /export failed/) + assert.deepEqual(exporter.stats(), { failures: 1, lastOkAt: 5_000, lastError: 'HTTP 503' }) +}) + +test('#P699 the health answer carries the export state', () => { + const window = { version: 1, now: 1_000, windowMs: 1_000, live: true, lanes: [], utilization: [], edges: [] } + const health = buildHealth(window, { failures: 3, lastOkAt: 900, lastError: 'HTTP 503' }) + assert.deepEqual(health.export, { enabled: true, failing: true, failures: 3, lastOkAt: 900, lastError: 'HTTP 503' }, 'the health answer must carry the export section the operator reads') +}) + +test('#P699 a host with no exporter says export is off, and a healthy one says it is healthy', () => { + const window = { version: 1, now: 1_000, windowMs: 1_000, live: true, lanes: [], utilization: [], edges: [] } + assert.deepEqual(buildHealth(window, null).export, { enabled: false }, 'a host with no exporter must say export is off') + assert.deepEqual(buildHealth(window, { failures: 0, lastOkAt: 1_000, lastError: null }).export, { enabled: true, failing: false, failures: 0, lastOkAt: 1_000, lastError: null }, 'a healthy exporter must say so') +}) + +test('#P699 three failed pushes behind the timer are one warning, and the endpoint names the streak', async () => { + const adapter = mockAdapter() + const timeline = createTimeline(adapter, { persist: false, now: () => 5_000, exporter: { send: collectorDown, intervalMs: 1e9 } }) + for (const record of records()) timeline.trace.append(record) + for (let attempt = 0; attempt < 3; attempt++) await timeline.exportNow() + const health = adapter.endpoints.get(HEALTH_ENDPOINT)({}) + timeline.dispose() + const exportWarns = adapter.warnings.filter(warning => warning.includes('export failed')) + assert.equal(exportWarns.length, 1, `a failing export warned ${exportWarns.length} time(s) in 3 pushes; a streak is one warning, not one per push`) + assert.ok(health.export, 'the health answer must carry an export section; it has none') + assert.deepEqual(health.export, { enabled: true, failing: true, failures: 3, lastOkAt: null, lastError: 'HTTP 503' }) +}) + +test('#P699 a 100ms interval against a dead collector is one warning in 560ms, not five', async () => { + const adapter = mockAdapter() + const timeline = createTimeline(adapter, { persist: false, now: () => 5_000, exporter: { send: collectorDown, intervalMs: 100 } }) + for (const record of records()) timeline.trace.append(record) + await new Promise(resolve => setTimeout(resolve, 560)) + const health = adapter.endpoints.get(HEALTH_ENDPOINT)({}) + timeline.dispose() + const exportWarns = adapter.warnings.filter(warning => warning.includes('export failed')) + assert.equal(exportWarns.length, 1, `a failing export warned ${exportWarns.length} time(s) in 560ms of a 100ms interval; a streak is one warning, not one per tick`) + assert.ok(health.export, 'the health answer must carry an export section; it has none') + assert.ok(health.export.failures >= 3, `the endpoint must count at least the 3 ticks that ran, counted ${health.export.failures}`) + assert.equal(health.export.failing, true) +})