From 9a635fc6336ceed1217f3d5042ad4e028b472134 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 4 Sep 2026 13:44:47 -0400 Subject: [PATCH 1/3] feat(console): net origin offload, and catch up to plugin 0.64.1; v0.12.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Origin offload is now stated twice. The existing figure — crawler requests not proxied live — is gross: it counts what the origin was spared and none of what this system asks of the origin in exchange. Net offload subtracts every origin request the deployment itself made, all from series already in the one analytics scan: proxied serves (bot_serve source=origin), renders (render/outcome, one row per posted result), change probes (probe_probed, a pass counter read by value) and sitemap fetches (sitemap_sitemaps). `originLoad` in charts.js carries the arithmetic so Traffic and the Overview cannot disagree; Traffic gains a "What the origin actually saw" panel with each term over time, the Overview tile carries the net figure under the gross one. Two things the number cannot know are stated on the panel, not assumed: a render counts as ONE origin request (the document), and the probe/sitemap counters land where a PASS FINISHED, so short ranges lump them — quote 24h. One term the plugin cannot see at all is exposed rather than omitted: whatever a crawler fetches from the origin after we hand it a page (a rendering crawler's scripts and the page's own API calls above all) never passes through this plugin. The net tile says "before crawler follow-up requests" and the panel reports every page handed to a crawler as the exposure — a count, never multiplied by a guessed factor, and for every crawler rather than a guessed subset of bots. The render fleet can measure the per-page factor; that is the follow-up. Catch-up to plugin 0.61–0.64.1: - `verified` (v0.63.0) is a cache serve. It sat outside every cache-served sum and fell into the "other" non-hit family; now one shared CACHE_SERVED set drives the freshness chart, the per-route table and the staleness sums, and it joins `invalidated` in one "Invalidation" family (one population: rescued and refused). - Invalidations gains a "what the invalidations are doing" panel reading page_verification, invalidation_reenqueue (incl. v0.64.0's forwarded / forward-failed — never added to lowered, the owner counts those) and invalidation_error (lkg-expired = failed OPEN). Two catalog-guard waivers removed; the guard's page_verification failure is what started this. - Change probe (v0.62.0): `stateAvailable: false` is named per node as "unknown, not idle"; a running sweep shows its heartbeat count. Co-Authored-By: Claude Code --- package-lock.json | 2 +- packages/console/README.md | 38 +++ packages/console/package.json | 2 +- packages/console/src/admin/charts.js | 172 +++++++++++ .../console/src/admin/views/invalidations.js | 281 ++++++++++++++++- packages/console/src/admin/views/overview.js | 19 +- packages/console/src/admin/views/probe.js | 59 +++- packages/console/src/admin/views/traffic.js | 220 +++++++++++-- packages/console/src/util/aggregate.js | 17 ++ packages/console/test/adminAssets.test.js | 5 +- packages/console/test/aggregate.test.js | 44 +++ .../console/test/invalidationsView.test.js | 289 ++++++++++++++++++ packages/console/test/overviewView.test.js | 52 +++- packages/console/test/probeView.test.js | 99 ++++++ packages/console/test/trafficView.test.js | 144 ++++++++- 15 files changed, 1405 insertions(+), 38 deletions(-) create mode 100644 packages/console/test/invalidationsView.test.js diff --git a/package-lock.json b/package-lock.json index e724345c..9b93a599 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8556,7 +8556,7 @@ }, "packages/console": { "name": "@harperfast/prerender-console", - "version": "0.11.0", + "version": "0.12.0", "license": "Apache-2.0", "dependencies": { "undici": "^7.18.2" diff --git a/packages/console/README.md b/packages/console/README.md index cd636485..5960b4e6 100644 --- a/packages/console/README.md +++ b/packages/console/README.md @@ -162,6 +162,37 @@ not have — the discovery gate, and a client-side bot filter), **Sitemaps**, ** **Queue** (render/claim health and the backlog), **Nodes**, **Invalidations** (preview-first record/clear), **Change probe**, **URL explainer**, **Metrics** (the live catalog), **Config**. +**Offload is stated twice, gross and net** (console v0.12.0). The gross figure — crawler requests +not proxied to the origin live — is what every serve-side panel is built from, and it is +flattering: it counts what the origin was spared and none of what this system asks of the origin +in exchange. Every render is the headless fleet loading a page from the origin; every change probe +is an origin call; every sitemap refresh fetches the sitemap. A deployment rendering its whole +corpus on a short cadence for a trickle of bot traffic can post a 95% gross offload while sending +the origin _more_ requests than the crawlers would have. Net offload is +`1 − (proxied + renders + probes + sitemap fetches) ÷ crawler requests arrived`, every term from a +series already in the one analytics scan, and the panel beside the origin-fetch chart shows each +term over time. Two caveats are written on the panel rather than assumed: a render counts as one +origin request (the document — the page's own subresources reach the origin only if the CDN does +not cache them for the renderer), and the probe and sitemap counters land in the bucket where a +_pass finished_, so a short range reads either none of a running sweep or all of one that just +ended — quote the 24h figure. One term the plugin cannot see at all is stated rather than +omitted: whatever a crawler fetches from the origin _after_ we hand it a page — a rendering +crawler's scripts and then the page's own XHR/API calls above all — never passes through this +plugin, so the net tile says "before crawler follow-up requests" and the panel reports the +exposure (every page handed to a crawler) as a count, never multiplied by a guessed factor. The +render fleet can measure the per-page factor; when a plugin reports it, that tile becomes a number. + +**Invalidations gained a third panel** for the same release: what the active rows are doing. +Refused serves (`invalidated` — each an origin round trip) beside rescued ones (`verified`, plugin +v0.63.0 — a page the change probe has _proved_ current since the epoch, served from cache through +the invalidation), the verifications the sweep recorded, and every outcome of the demand-driven +heal including v0.64.0's cross-node `forwarded` / `forward-failed`. `forwarded` is deliberately +never added to `lowered`: the owner counts a forwarded heal under its own verdict in the same +series, so adding the two double-counts under cluster scope. The `verified` status is also a +cache serve everywhere else in the console — the freshness chart, the per-route table, the +staleness sums — and sits in one "Invalidation" family with `invalidated` on the non-hit strip, +because the two are one population split into rescued and refused. + **Change probe is new** (plugin v0.53.0+), and it is the one freshness surface that does not measure pages against a cadence. It reads what the probe is actually detecting — the change rate against the probes that HAD a baseline, not against every probe, because a pass that is mostly seeding has @@ -182,6 +213,13 @@ flagged when a settled deployment skips most of them, which means `reprobeAfter` `sweepInterval`), and registry rows that could not be decoded, which is a storage-layer escalation rather than anything a setting here reaches. +Plugin v0.62.0 moved the probe's state into a node-local row every worker can read (before it, 15 +of 16 workers answered "not running, never ran" — indistinguishable from the probe being off), and +the endpoint now says when that row could _not_ be read. The console prints that as the loudest +note on the page — the node is unknown, not idle — and shows a running sweep's heartbeat count +("~24,000 rows examined") on the header pill, the sweep card and the per-node table, summed across +the nodes running because each walks its own slice. + Plugin v0.58.0's `pageCheck` adds the one counter on that card that is not about the origin changing: **page mismatches** — cached pages that disagree with the origin on a field they claim. That is the class the signature comparison is structurally blind to (a value that changed and diff --git a/packages/console/package.json b/packages/console/package.json index 4f994f78..62f62687 100644 --- a/packages/console/package.json +++ b/packages/console/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-console", - "version": "0.11.0", + "version": "0.12.0", "type": "module", "description": "Standalone Harper component serving the prerender management console UI, proxying to a prerender deployment's /prerender_admin API", "license": "Apache-2.0", diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index 7da1d8d0..3cae3502 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -40,6 +40,10 @@ export const CACHE_STATUS_COLORS = { 'hit': OK, 'swr': INFO, 'peer-rescue': '#8ff1cd', + // A cache serve an invalidation would have refused, let through on the probe's evidence + // (plugin v0.63.0). Coloured beside `invalidated` rather than beside `hit`: the two are the + // same population — pages a bulk invalidation touched — split into rescued and refused. + 'verified': '#c9a7ff', 'miss': WARN, 'stale': PINK, 'invalidated': PURPLE, @@ -49,6 +53,19 @@ export const CACHE_STATUS_COLORS = { 'blob-timeout': BAD, }; +/** + * The freshness verdicts that are CACHE SERVES — the bytes came from a stored snapshot. This is + * the one enumeration every "cache-served" sum in the console must share, because a verdict + * missing from it does not read as missing: it reads as a smaller hit rate. `verified` sat outside + * every such sum for one plugin release and the cache-served share simply looked lower. + * + * `hit`, `swr` and `verified` are the page itself; `peer-rescue` is the owner's copy of it. What + * is NOT here: `miss`/`stale` (origin), `blob-*` (the local body failed AND no rescue landed, so + * the request went to origin), `invalidated` (refused), `skip`/`bypass` (never consulted). + */ +export const CACHE_SERVED = new Set(['hit', 'swr', 'verified', 'peer-rescue']); +export const isCacheServed = (status) => CACHE_SERVED.has(status); + /** Where the bytes came from (bot_serve.path). `origin` is the one offload counts against. */ export const SOURCE_COLORS = { cache: OK, rendered: INFO, origin: WARN }; @@ -534,3 +551,158 @@ export const emptyNote = (what, data) => el('code', { text: 'analytics.enabled' }), ' gates recording). Analytics rows are node-local; a cluster view sums every node’s own slice.', ]); + +// ---- origin load: the net offload arithmetic --------------------------------- +// +// Shared by Traffic (which shows every term) and the Overview (which shows the figure), so the two +// can never disagree about what "net" means. + +// Series names as constants: the route-contract scanner in adminAssets.test.js reads a quoted +// name inside a Map lookup or a comparison as a fetch of a route by that name. +const RENDER_OUTCOME = 'outcome'; +const PROBE_REQUESTS = 'probe_probed'; +const SITEMAP_FETCHES = 'sitemap_sitemaps'; + +/** + * Every request the origin answered because this deployment exists, over the window, beside the + * requests crawlers made — the two sides of the net offload figure. + * + * GROSS OFFLOAD IS FLATTERING. It is the share of crawler requests that were not proxied live + * (`bot_serve` source ≠ origin), and it counts what the origin was spared and none of what this + * system asks of the origin in exchange. A deployment rendering its whole corpus on a short + * cadence for a trickle of bot traffic can post a 95% gross offload while sending the origin MORE + * requests than the crawlers would have. Net offload is + * `1 − (proxied + renders + probes + sitemap fetches) ÷ crawler requests arrived`. + * + * FOUR TERMS, each from a series already in the scan: + * + * proxied `bot_serve` source = origin. A crawler request forwarded live: a miss, a page past + * its SWR window, a blob fault nobody rescued, an invalidated page, a skip, a bypass. + * renders `render` series `outcome` — EXACTLY one row per posted result (metrics.js), so its + * count is the number of page loads the headless fleet performed against the origin, + * whatever became of each (rendered, suppressed, failed, transient…). Each is counted as + * ONE origin request: the document. A page load also pulls the page's own scripts and + * stylesheets, which reach the origin only if the CDN does not cache them for the + * renderer — a deployment fact no metric here can see, so it is stated, not estimated. + * probes `probe_probed` — one origin call per attempt, failures included (a refused probe was + * still a request). Sweep and canary both emit it. A probe hits a small endpoint, not a + * page render, so it is cheaper than the other three; it is still a request. + * sitemaps `sitemap_sitemaps` — one fetch per sitemap a refresh run processed. + * + * `renders` is read with sumCount (one emit = one result); the two pass counters with sumValues + * (one emit per pass carrying the pass's count — sumCount there would count passes). + * + * THE DENOMINATOR IS `bot_request`, not `bot_serve`: without this system the CDN forwards every + * crawler request to the origin, including the few that never reach a serve outcome here, so the + * baseline is what ARRIVED. When the window has serves but no requests (it should not — both are + * gated identically — but an older plugin could), the serve total stands in. + * + * A FIFTH TERM IS EXPOSED BUT NOT COUNTED — what a crawler fetches from the origin AFTER we hand + * it a page. A rendering crawler (Google's WRS, Bingbot, Applebot) loads the page's scripts and + * then makes whatever XHR/fetch calls the page makes — inventory, pricing, personalisation — and + * those are the calls least likely to be cacheable anywhere; an image crawler fetches the images; + * any crawler may follow a linked resource. None of it passes through this plugin: the CDN + * forwards the DOCUMENT request to us and sends the crawler's subrequests straight to the origin, + * so no series here can count them, and a net figure that quietly omitted them would be + * flattering in exactly the way the gross one is. What CAN be counted is the exposure: every page + * handed to a crawler — ALL of them, not a guessed subset of bots that "probably" fetch more, + * because which crawler does what is a fact about the crawler that this console should not + * assert. It is reported as that — a count of pages, never multiplied by a guessed + * requests-per-page — and every reader of this figure says "before crawler follow-up requests". + * The measured version needs the render fleet and the registry: our own headless Chrome loads the + * same pages and already runs a cache policy on every same-origin response, so "uncacheable + * same-origin subrequests per page load" is measurable there; stored on the cached page and + * multiplied at serve time by what the registry says that crawler fetches, it becomes a counted + * term. A snapshot with its scripts stripped hydrates nothing, which makes even the rendering + * crawlers' share of this an upper bound. + * + * WHAT IS NOT COUNTED, so nobody assumes it is: crawler requests the CDN answered from its own + * cache (they never reach this plugin), renders that crashed before posting a result (no row), and + * cluster-internal traffic — peer rescue and forwarded heals are node-to-node, never origin. + * + * `lumpy` is true when either pass counter contributed: those land in the bucket where the pass + * FINISHED, so over a range shorter than a pass the term is either absent or all of it. Over a + * 24h range it is right to within one pass. + */ +export function originLoad(data) { + const serves = pick(data, 'bot_serve'); + const served = sumCount(serves); + const proxied = sumCount(serves.filter((s) => s.path === 'origin')); + const renders = sumCount(pick(data, 'render', (s) => s.path === RENDER_OUTCOME)); + const probes = sumValues(pick(data, 'prerender_ops', (s) => s.path === PROBE_REQUESTS)); + const sitemaps = sumValues(pick(data, 'prerender_ops', (s) => s.path === SITEMAP_FETCHES)); + const requests = sumCount(pick(data, 'bot_request')); + const arrived = requests > 0 ? requests : served; + const total = proxied + renders + probes + sitemaps; + + return { + proxied, + renders, + probes, + sitemaps, + total, + arrived, + // Null, never 0 or 100%: an empty window has no offload to report. + net: arrived > 0 ? (arrived - total) / arrived : null, + lumpy: probes > 0 || sitemaps > 0, + // The fifth term's exposure: every page handed to a crawler, cache-served or proxied alike — + // the crawler fetches what it fetches next either way. + handed: served, + }; +} + +/** + * Per-bucket origin load by cause, for a stacked chart: stackBy's inner loop applied per TERM + * rather than over one metric's dimension, because the four terms come from three metrics and two + * value semantics. The proxied and render terms are per-request counts; the two pass counters are + * VALUES (one emit carrying a count), so their per-bucket form is mean × count, as in stackBy's + * values mode. + */ +export function originLoadBuckets(data) { + const bucketsOf = (combos, values) => { + const out = new Array(data.bucketCount).fill(0); + for (const combo of combos) { + for (let i = 0; i < combo.counts.length && i < out.length; i++) { + if (!values) out[i] += combo.counts[i]; + else if (Number.isFinite(combo.means?.[i]) && combo.counts[i] > 0) out[i] += combo.means[i] * combo.counts[i]; + } + } + return out; + }; + return new Map([ + [ + 'proxied', + bucketsOf( + pick(data, 'bot_serve', (s) => s.path === 'origin'), + false + ), + ], + [ + 'renders', + bucketsOf( + pick(data, 'render', (s) => s.path === RENDER_OUTCOME), + false + ), + ], + [ + 'probes', + bucketsOf( + pick(data, 'prerender_ops', (s) => s.path === PROBE_REQUESTS), + true + ), + ], + [ + 'sitemaps', + bucketsOf( + pick(data, 'prerender_ops', (s) => s.path === SITEMAP_FETCHES), + true + ), + ], + ]); +} + +/** + * A net offload as a percentage that may be NEGATIVE. `pct()` is built for shares of a whole; + * this is a signed ratio of 1, and a minus sign here is the finding. + */ +export const fmtNet = (net) => (Number.isFinite(net) ? `${Math.round(net * 100)}%` : '—'); diff --git a/packages/console/src/admin/views/invalidations.js b/packages/console/src/admin/views/invalidations.js index 81634a20..2b29c11e 100644 --- a/packages/console/src/admin/views/invalidations.js +++ b/packages/console/src/admin/views/invalidations.js @@ -13,16 +13,45 @@ * AN UNRESOLVABLE ROW IS THE LOUDEST THING ON THE PAGE. A row whose scope no longer names * a configured route looks applied and matches nothing — worse than no row — so it gets * the bad pill, not a footnote. + * + * WHAT AN INVALIDATION IS DOING is the third panel, and it is the one that needs the analytics + * window. A row on the table says what is invalidated; it says nothing about what that costs + * (every refused serve is an origin round trip) or what the two mechanisms built to cut that + * cost are buying back — per-page verification (plugin v0.63.0: a page the probe has PROVED + * current is served through the invalidation, `bot_serve` cacheStatus `verified`) and the + * demand-driven heal (a crawled invalidated page is pulled forward in the queue, with v0.64.0 + * forwarding the heal to the key's owner instead of refusing it as `not-owner`). Those are + * counters, they live in the analytics scan, and this is the view an operator is on when they + * ask "is it working". The read is the same one-hour window the overview already holds, so on a + * warm cache it costs no scan at all. */ -import { ago, card, el, ICONS, kv, mono, muted, num, pill, spacer, table } from '../ui.js'; -import { appliedNote, editTray, loadConfig, settingsCard } from './_configEdit.js'; +import { ago, card, el, ICONS, kv, mono, muted, num, pct, pill, spacer, stat, table } from '../ui.js'; +import { + barList, + emptyNote, + fmtCount, + pick, + scanFooter, + scopeLabel, + SERIES, + sumCount, + windowEmpty, +} from '../charts.js'; +import { appliedNote, configState, editTray, loadConfig, optionIndex, settingsCard } from './_configEdit.js'; export const meta = { id: 'invalidations', label: 'Invalidations', crumb: 'invalidations', icon: ICONS.invalidations }; export async function load(ctx) { - const [res] = await Promise.all([ctx.get('invalidations'), loadConfig(ctx)]); + const [res, analyticsRes] = await Promise.all([ + ctx.get('invalidations'), + // The same range key Overview, Queue and Nodes use, so this is served from the worker's + // cache whenever any of them loaded inside management.analytics.cacheTtl. + ctx.get('analytics', { range: 3_600_000 }), + loadConfig(ctx), + ]); ctx.data.list = res.ok ? res.body : null; + ctx.data.analytics = analyticsRes.ok ? analyticsRes.body : null; ctx.data.error = res.ok ? null : (res.body?.error ?? `Could not load invalidations (${res.status})`); } @@ -45,12 +74,249 @@ export function render(ctx) { ' is false — they are recorded and NOT enforced. Whatever those rows were protecting against is being served.', ]), active(ctx, data), + effect(ctx, data), record(ctx, data), settings(ctx), editTray(ctx), ]; } +// ---- what the invalidations are doing --------------------------------------- + +// Series and outcome names as constants: the route-contract test in adminAssets.test.js reads a +// quoted name inside a Map lookup as a call to an admin route of that name. +const PAGE_VERIFICATION = 'page_verification'; +const REENQUEUE = 'invalidation_reenqueue'; +const INVALIDATION_ERROR = 'invalidation_error'; +const VERIFIED = 'verified'; +const INVALIDATED = 'invalidated'; +const WRITTEN = 'written'; +const LOWERED = 'lowered'; +const NOT_OWNER = 'not-owner'; +const LKG_EXPIRED = 'lkg-expired'; + +/** + * Every outcome the heal can record, with what it means — the closed set the plugin exports as + * `REENQUEUE_OUTCOMES`. Order is the order an operator should read them in: what worked, what was + * handed on, then why the rest did not, most actionable first. + * + * `forwarded` and `lowered` are deliberately never added together: a forwarded heal is counted + * again by the OWNER under its own verdict in this same series, so under cluster scope `lowered` + * already includes the forwarded heals that landed and adding `forwarded` would double-count them. + */ +const HEAL_OUTCOMES = [ + { + key: 'lowered', + label: 'lowered', + tone: 'ok', + means: 'the URL’s due time was pulled forward — it heals ahead of its cadence', + }, + { + key: 'forwarded', + label: 'forwarded', + tone: 'info', + means: 'this node did not own the key and handed the heal to the owner, which counts its own verdict here', + }, + { + key: 'not-owner', + label: 'not-owner', + tone: 'warn', + means: 'this node did not own the key and cross-node forwarding is off — the heal was dropped', + }, + { + key: 'forward-failed', + label: 'forward-failed', + tone: 'bad', + means: 'the owner could not be reached in time — a transport fault, not a refusal', + }, + { key: 'throttled', label: 'throttled', tone: 'warn', means: 'invalidation.reenqueue.maxPerMinute was spent' }, + { key: 'not-sooner', label: 'not-sooner', tone: '', means: 'already due at least as soon — nothing to pull forward' }, + { + key: 'unhealable', + label: 'unhealable', + tone: '', + means: 'already rendered after the epoch, or striking out — a re-render would not help', + }, + { key: 'leased', label: 'leased', tone: '', means: 'a render is in flight for this key right now' }, + { key: 'paused', label: 'paused', tone: '', means: 'the queue is paused on this node' }, + { + key: 'no-schedule', + label: 'no-schedule', + tone: 'bad', + means: 'a live URL with no schedule row — the terminal gap render.reconcile repairs', + }, + { key: 'no-target', label: 'no-target', tone: '', means: 'no Target for the URL — nothing to schedule' }, + { key: 'error', label: 'error', tone: 'bad', means: 'the schedule write threw' }, +]; + +/** Total per `dim` value over the given combos, as an ordered list. */ +function tally(combos, dim) { + const totals = new Map(); + for (const s of combos) totals.set(s[dim] ?? 'unknown', (totals.get(s[dim] ?? 'unknown') ?? 0) + s.count); + return totals; +} + +function effect(ctx, list) { + const data = ctx.data.analytics; + const title = data + ? `What the invalidations are doing — ${scopeLabel(data)}, last hour` + : 'What the invalidations are doing'; + const options = optionIndex(configState(ctx).payload); + const on = (path) => options.get(path)?.effective === true; + const verificationOn = on('invalidation.verification.enabled'); + const reenqueueOn = on('invalidation.reenqueue.enabled'); + const crossNodeOn = on('invalidation.reenqueue.crossNode.enabled'); + + if (!data || data.available === false || windowEmpty(data)) { + return card(title, { body: [emptyNote('bot_serve', data)] }); + } + + const serves = pick(data, 'bot_serve'); + const refused = sumCount(serves.filter((s) => s.method === INVALIDATED)); + const rescued = sumCount(serves.filter((s) => s.method === VERIFIED)); + const touched = refused + rescued; + + const writes = tally( + pick(data, 'prerender_ops', (s) => s.path === PAGE_VERIFICATION), + 'method' + ); + const written = writes.get(WRITTEN) ?? 0; + const writeFaults = [...writes.entries()].filter(([outcome]) => outcome !== WRITTEN); + + const heals = tally( + pick(data, 'prerender_ops', (s) => s.path === REENQUEUE), + 'method' + ); + const healTotal = [...heals.values()].reduce((acc, n) => acc + n, 0); + const healRows = HEAL_OUTCOMES.filter((o) => heals.has(o.key)).map((o) => ({ ...o, count: heals.get(o.key) })); + // An outcome this console has no entry for still gets a row, uncoloured, rather than vanishing + // from a list whose whole point is that it sums to the attempts. + for (const [outcome, count] of heals) { + if (!HEAL_OUTCOMES.some((o) => o.key === outcome)) { + healRows.push({ + key: outcome, + label: outcome, + tone: '', + means: 'an outcome this console does not know about', + count, + }); + } + } + const notOwner = heals.get(NOT_OWNER) ?? 0; + + const errors = tally( + pick(data, 'prerender_ops', (s) => s.path === INVALIDATION_ERROR), + 'method' + ); + const lkgExpired = errors.get(LKG_EXPIRED) ?? 0; + const errorTotal = [...errors.values()].reduce((acc, n) => acc + n, 0); + + const nothingActive = !(list.invalidations?.length > 0); + + return card(title, { + head: [ + spacer(), + verificationOn ? pill('verification on', 'ok') : pill('verification off', ''), + reenqueueOn ? pill(crossNodeOn ? 'heal on · cross-node' : 'heal on', 'ok') : pill('heal off', ''), + ], + body: [ + el('div', { cls: 'stat-grid tight' }, [ + stat( + 'Refused', + num(refused), + touched + ? `${pct(refused, touched)} of touched serves · each an origin round trip` + : 'serves an invalidation cost', + { warn: refused > 0 && rescued === 0 && verificationOn } + ), + stat( + 'Rescued', + num(rescued), + touched + ? `${pct(rescued, touched)} · served on the probe’s evidence (verified)` + : 'served through an invalidation on evidence' + ), + stat( + 'Verifications recorded', + num(written), + writeFaults.length + ? `${writeFaults.map(([outcome, count]) => `${num(count)} ${outcome}`).join(' · ')}` + : 'by the probe sweep, while a scope is invalidated', + { warn: writeFaults.length > 0 } + ), + stat( + 'Heals attempted', + num(healTotal), + healTotal + ? `${num(heals.get(LOWERED) ?? 0)} lowered · ${pct(notOwner, healTotal)} not-owner` + : 'crawled invalidated pages pulled forward', + // The accelerator's real ceiling: traffic lands where the CDN sends it while residency + // is hashed, so on a concentrated deployment most heals arrive off-owner. Past half, + // cross-node forwarding is the lever. + { warn: healTotal > 0 && notOwner > healTotal / 2 && !crossNodeOn } + ), + ]), + healRows.length + ? barList( + healRows.map((row) => ({ + label: row.label, + value: row.count, + color: + row.tone === 'ok' + ? 'var(--ok)' + : row.tone === 'bad' + ? 'var(--bad)' + : row.tone === 'warn' + ? 'var(--warn)' + : row.tone === 'info' + ? 'var(--info)' + : SERIES[0], + sub: pct(row.count, healTotal), + title: `${row.label}: ${num(row.count)} — ${row.means}`, + })), + { format: fmtCount } + ) + : null, + errorTotal > 0 && + el('div', { cls: lkgExpired > 0 ? 'note bad' : 'note warn' }, [ + el('strong', { text: `${num(errorTotal)} epoch resolution failure(s) this hour: ` }), + [...errors.entries()].map(([kind, count]) => `${kind} ${num(count)}`).join(' · '), + lkgExpired > 0 + ? '. lkg-expired means the row read threw AND the last-known-good memory was older than invalidation.lkgMaxAge, so those requests failed OPEN — content someone deliberately invalidated may have been served.' + : '. read-error means a live last-known-good answered; invalid-row and unknown-mode were treated as hard invalidations.', + ]), + nothingActive && + !touched && + el('div', { cls: 'note' }, [ + 'Nothing is invalidated right now, so none of this is expected to move: a verification is only ', + 'written while a scope is invalidated, and a heal only fires on a serve an invalidation refused. ', + 'Zeros here are the steady state, not a fault.', + ]), + !nothingActive && + touched > 0 && + !verificationOn && + el('div', { cls: 'note' }, [ + 'Every touched serve was refused because ', + el('code', { text: 'invalidation.verification.enabled' }), + ' is off. With it on, a page the change probe has re-confirmed against the origin since the epoch is ', + 'served from cache instead of proxied — measured at 71-78% of a route-wide invalidation’s scope on ', + 'one deployment. It requires ', + el('code', { text: 'changeProbe.pageCheck' }), + ' on the rule whose scope was invalidated.', + ]), + el('p', { cls: 'muted chart-note' }, [ + 'Refused and rescued are the population an invalidation touched — pages rendered before the epoch that ', + 'a crawler asked for — split by what happened next. A refused serve is proxied to the origin (it is ', + 'also inside the miss/origin figures on Traffic); a rescued one is served from cache because the probe ', + 'proved its price and availability current, which is all “verified” asserts. The heal outcomes sum to ', + 'attempts; “forwarded” is a hand-off whose result the owner counts under its own verdict, so it is ', + 'never added to “lowered”. Counters, node-local, summed under cluster scope.', + ]), + ], + foot: [scanFooter(data)], + }); +} + function active(ctx, data) { const rows = (data.invalidations ?? []).map((row) => el('tr', null, [ @@ -211,7 +477,10 @@ const settings = (ctx) => 'How the rows in Active invalidations above are applied. enabled is the header pill: false leaves ' + 'every row stored and stops honouring all of them at once, so the corpus serves pre-invalidation ' + 'bytes again — it does not clear anything. maxScopes is the slot count on that panel. pad only ' + - 'widens the comparison toward invalidating (its cost is at most one extra render per page), and the ' + - 'reenqueue options decide whether a bot request for an invalidated page pulls that URL forward in ' + - 'the queue instead of waiting out its own cadence. Nothing here rewrites a cached page.', + 'widens the comparison toward invalidating (its cost is at most one extra render per page). ' + + 'verification lets a page the change probe has PROVED current since the epoch be served through ' + + 'the invalidation (the “rescued” count above; needs pageCheck on the rule). The reenqueue options ' + + 'decide whether a bot request for an invalidated page pulls that URL forward in the queue instead of ' + + 'waiting out its own cadence, and crossNode whether a heal for a key another node owns is forwarded ' + + 'to it rather than dropped as not-owner. Nothing here rewrites a cached page.', }); diff --git a/packages/console/src/admin/views/overview.js b/packages/console/src/admin/views/overview.js index 0389863e..83de21ad 100644 --- a/packages/console/src/admin/views/overview.js +++ b/packages/console/src/admin/views/overview.js @@ -42,8 +42,10 @@ import { colorFor, emptyNote, fmtMs, + fmtNet, isMerged, legend, + originLoad, pick, scopeLabel, stackBy, @@ -210,6 +212,9 @@ function traffic(ctx) { const total = sumCount(serves); const originServes = sumCount(serves.filter((s) => s.path === 'origin')); const cacheServes = sumCount(serves.filter((s) => s.path === 'cache')); + // The figure the gross tile flatters: renders, probes and sitemap fetches are origin requests + // too. Same arithmetic as Traffic (charts.js), so the two views cannot disagree about "net". + const load = originLoad(data); // The median, matching Traffic: an evenly refreshed corpus puts its p95 within a whisker of the // interval by construction, so a p95 tile here would read as "behind" on a healthy fleet. const ageMedian = weighted(pick(data, 'page_age'), 'median'); @@ -226,9 +231,17 @@ function traffic(ctx) { body: [ el('div', { cls: 'stat-grid tight' }, [ stat('Serves', num(total)), - stat('Origin offload', pct(total - originServes, total), null, { - warn: total > 0 && originServes > total / 2, - }), + stat( + 'Origin offload', + pct(total - originServes, total), + // Gross on the face, net underneath — the subtitle is what stops a 90% headline being + // quoted for a deployment whose renders and probes hand most of it back. + Number.isFinite(load.net) + ? `${fmtNet(load.net)} net of renders + probes · before crawler follow-up requests` + : 'gross — crawler requests not proxied live', + // Either figure under half is the flag; the net one is the one that can go negative. + { warn: (total > 0 && originServes > total / 2) || (Number.isFinite(load.net) && load.net < 0.5) } + ), stat('Cache-served', pct(cacheServes, total)), stat('Page age', fmtMs(ageMedian), 'median, cache serves only ≈', { warn: Number.isFinite(ageMedian) && Number.isFinite(interval) && ageMedian > interval, diff --git a/packages/console/src/admin/views/probe.js b/packages/console/src/admin/views/probe.js index 589dc1cf..bea46569 100644 --- a/packages/console/src/admin/views/probe.js +++ b/packages/console/src/admin/views/probe.js @@ -95,6 +95,29 @@ const sweepCadence = (sweep) => { return `continuous · ${target} · ${slice}`; }; +/** + * Nodes whose probe state row could not be read (plugin v0.62.0's `stateAvailable: false`), + * normalized over the node and cluster payload shapes. Empty for an older plugin, which has no + * such row and no such field. + */ +const stateUnavailableOn = (status) => + status.stateUnavailableOn ?? (status.stateAvailable === false ? [status.node ?? 'this node'] : []); + +/** + * "sweep running", with how far it has got when the plugin says. `examinedApprox` is the pass's + * heartbeat counter — rows walked so far, rounded to the yield interval — and it is the one number + * that separates a sweep that is moving from one that merely claims to be. Summed across the + * nodes running, because each walks its own disjoint slice. + */ +const sweepProgress = (status) => { + // One object at node scope, a per-node list under the cluster merge. + const progress = status.sweep?.progress; + const list = Array.isArray(progress) ? progress : progress ? [progress] : []; + const examined = list.reduce((acc, p) => acc + (Number.isFinite(p?.examinedApprox) ? p.examinedApprox : 0), 0); + return examined > 0 ? ` · ~${num(examined)} rows examined` : ''; +}; +const sweepRunningLabel = (status) => `sweep running${sweepProgress(status)}`; + /** * The pass counters, in the order a pass produces them. * @@ -236,6 +259,24 @@ function state(ctx, status) { ); } + // Plugin v0.62.0 publishes the probe's state to a node-local row so any worker can answer for + // the node; before it, 15 of 16 workers answered "not running, never ran" — indistinguishable + // from the probe being off. `stateAvailable: false` is the row failing to READ, and everything + // this page says about that node's sweep and canary is then untrustworthy rather than + // reassuring. Above the counters, because it changes what they mean. + const stateUnavailable = stateUnavailableOn(status); + if (stateUnavailable.length) { + body.push( + note('bad', [ + `The change-probe state row could not be read on ${stateUnavailable.join(', ')}. `, + 'Everything this page reports for ', + stateUnavailable.length > 1 ? 'those nodes' : 'that node', + ' — running or not, last pass, armed cadence — is unknown, not idle: the sweep may well be running. ', + 'The node’s log says why the read failed (a coordination database problem, usually).', + ]) + ); + } + body.push( kv([ ['Sweep', sweepCadence(status.sweep) ?? 'not armed'], @@ -289,7 +330,7 @@ function state(ctx, status) { : 'live', 'ok' ), - status.sweep?.running && pill('sweep running', 'info'), + status.sweep?.running && pill(sweepRunningLabel(status), 'info'), status.canary?.running && pill('canary running', 'info'), spacer(), segmented( @@ -697,7 +738,10 @@ function sweepCard(ctx, status) { head: [ sweepCadence(sweep) ? pill(sweepCadence(sweep), 'ok') : pill('not armed', 'bad'), sweep.running && - pill(sweep.runningOn?.length ? `running on ${sweep.runningOn.join(', ')}` : 'running now', 'info'), + pill( + `${sweep.runningOn?.length ? `running on ${sweep.runningOn.join(', ')}` : 'running now'}${sweepProgress(status)}`, + 'info' + ), spacer(), muted(last?.nodes > 1 ? `summed over ${last.nodes} nodes’ own slices` : ''), ], @@ -855,8 +899,17 @@ function nodeTable(status) { el('tr', null, [ el('td', { cls: 'mono', text: row.hostname }), el('td', null, [ + // An unreadable state row outranks everything else in the cell: "enabled" is read + // from config and is true, but running/last-pass/probed are all unknown for this node. + row.stateAvailable === false && pill('state unreadable', 'bad'), row.enabled === false ? pill('disabled', 'bad') : pill('enabled', 'ok'), - row.sweepRunning && pill('sweeping', 'info'), + row.sweepRunning && + pill( + Number.isFinite(row.sweepProgress?.examinedApprox) + ? `sweeping · ~${num(row.sweepProgress.examinedApprox)} examined` + : 'sweeping', + 'info' + ), row.canaryRunning && pill('canary', 'info'), ]), el('td', null, [row.dryRun === false ? pill('live', 'ok') : pill('dry run', 'warn')]), diff --git a/packages/console/src/admin/views/traffic.js b/packages/console/src/admin/views/traffic.js index 9f42d82f..f173f587 100644 --- a/packages/console/src/admin/views/traffic.js +++ b/packages/console/src/admin/views/traffic.js @@ -18,6 +18,18 @@ * it costs no request), falling back to `render.defaultInterval`. Absolute milliseconds stay one * click away, and the KPI carries both. * + * OFFLOAD IS STATED TWICE, GROSS AND NET, because the number the rollout is judged on is not the + * one the serve path can see. Gross offload is the share of crawler requests that were not proxied + * to the origin live — `bot_serve` source ≠ origin — and it is what every serve-side panel here is + * built from. It is also flattering: it counts what the origin was spared and none of what this + * system asks of the origin in exchange — every render, every change probe, every sitemap fetch is + * an origin request. Net offload subtracts those (`originLoad` in charts.js carries the arithmetic + * and its caveats), and the panel beside the origin-fetch chart shows each term. One term it + * cannot count is stated rather than omitted: whatever a crawler fetches from the origin after we + * hand it a page (a rendering crawler's scripts and API calls above all), which never passes + * through this plugin. The net tile says "before crawler follow-up requests" and the panel reports + * the exposure — every page handed to a crawler — as a count, never multiplied by a guessed factor. + * * AND NOT EVERY MISS IS OURS. A miss whose origin fetch came back 404 or 410 is a URL that does * not exist anywhere — there is nothing for the cache to be missing, and it can never improve, * because only a 200 is ever scheduled for prerendering. Left in the coverage number those URLs @@ -39,9 +51,11 @@ * THE BOT FILTER IS CLIENT-SIDE, ALWAYS. Selecting bots re-renders from the payload already in * hand; it never refetches, because the load discipline below is the whole reason this view can * afford to be this detailed. Four metrics carry a bot (`bot_request`, `bot_serve`, `page_age`, - * and `prerender_ops`'s `discovery_gated`) and the rest do not — `route_serve`, `origin_fetch`, `duration` and `response_*` have no bot - * dimension at all — so a panel that CANNOT honour the filter says "all bots" on its face rather - * than quietly showing every crawler's numbers under one crawler's name. + * and `prerender_ops`'s `discovery_gated`) and the rest do not — `route_serve`, `origin_fetch`, + * `render`, the probe and sitemap counters, `duration` and `response_*` have no bot dimension at + * all — so a panel that CANNOT honour the filter says "all bots" on its face rather than quietly + * showing every crawler's numbers under one crawler's name. Net offload is one of those: a render + * is not for any one crawler. * * LOAD DISCIPLINE. Every number on this view comes from a single `analytics` request; each * node answers it from a per-worker cache inside `management.analytics.cacheTtl`, so switching @@ -69,9 +83,13 @@ import { emptyNote, fmtCount, fmtMs, + fmtNet, fmtRatio, + isCacheServed, legend, lineChart, + originLoad, + originLoadBuckets, pick, rangePicker, ratioOf, @@ -148,7 +166,9 @@ export function render(ctx) { const requests = pick(data, 'bot_request', (s) => keepBot(filter, s.method)); const ages = pick(data, 'page_age', (s) => keepBot(filter, s.path)); const cadences = cadenceIndex(configState(ctx).payload, data.intervals?.defaultRenderInterval); - const scope = { serves, requests, ages, cadences, filter }; + // Over ALL bots whatever the filter says: three of its four terms carry no bot (see the header). + const load = originLoad(data); + const scope = { serves, requests, ages, cadences, filter, load }; return [ head, @@ -158,8 +178,10 @@ export function render(ctx) { kpis(data, scope), el('div', { cls: 'cols' }, [freshness(data, scope), staleness(ctx, data, scope)]), notFreshHit(data, scope), - el('div', { cls: 'cols' }, [originFetch(data, filter), latency(data, filter)]), - el('div', { cls: 'cols' }, [crawlers(data, scope), statusCodes(data, filter)]), + // The two origin-side panels together: what the origin was asked, then what each ask cost. + el('div', { cls: 'cols' }, [originSeen(data, scope), originFetch(data, filter)]), + el('div', { cls: 'cols' }, [latency(data, filter), statusCodes(data, filter)]), + crawlers(data, scope), routes(ctx, data, cadences, filter), discoveryGate(ctx, data, filter), breadth(ctx, filter), @@ -377,6 +399,7 @@ function kpis(data, scope) { const freshHits = sumCount(serves.filter((s) => s.method === 'hit')); const coverage = coverageSplit({ serves, costs: originCostByReason(data), filter }); const arrived = sumCount(requests); + const { load } = scope; // SERVE TIME IS TWO POPULATIONS, and one number over both is a number about the hit rate. A // cache hit is single-digit milliseconds and an origin proxy is hundreds; pooling them produces @@ -420,12 +443,27 @@ function kpis(data, scope) { { warn: unresolvedShare > 0.02 } ), stat( - 'Origin offload', + 'Origin offload · gross', pct(total - originServes, total), - 'requests the origin never saw', + 'crawler requests not proxied live', // The offload number is the rollout's headline; a majority-origin window deserves the flag. { warn: total > 0 && originServes > total / 2 } ), + stat( + 'Origin offload · net', + fmtNet(load.net), + // The subtitle carries the sum, because the tile's whole point is what the gross figure + // leaves out — and under a bot filter it says so, since this one cannot narrow. "Before + // crawler follow-up requests" is the term that is NOT in the sum (see originLoad). + load.arrived > 0 + ? `origin saw ${fmtCount(load.total)} of ${fmtCount(load.arrived)} asked · before crawler follow-up requests${ + filter ? ' · all bots' : '' + }` + : 'no crawler requests in the window', + // Below half is a flag on either figure; below ZERO means this system is sending the origin + // more requests than the crawlers would have on their own — the finding, not a display bug. + { warn: Number.isFinite(load.net) && load.net < 0.5 } + ), stat('Cache-served', pct(cacheServes, total), 'stored snapshot answered'), stat('Fresh hits', pct(freshHits, total), 'inside the configured cadence'), stat( @@ -473,9 +511,9 @@ function freshness(data, { serves, filter }) { ? stackedBars(data, keys, stacks, (k) => colorFor(CACHE_STATUS_COLORS, k)) : emptyNote('bot_serve', data), el('p', { cls: 'muted chart-note' }, [ - 'hit + swr + peer-rescue is cache-served. A rising miss share is a coverage problem; a rising ', - 'swr share is the fleet not keeping the configured cadence; blob-* should sit at zero. What each ', - 'verdict costs, and which of them are the same problem, is the panel below.', + 'hit + swr + verified + peer-rescue is cache-served. A rising miss share is a coverage problem; a ', + 'rising swr share is the fleet not keeping the configured cadence; blob-* should sit at zero. What ', + 'each verdict costs, and which of them are the same problem, is the panel below.', ]), filter && muted(`Filtered to ${[...filter].join(', ')}.`), ], @@ -534,9 +572,7 @@ function staleness(ctx, data, scope) { // differ whenever a target's cadence comes from its stored value (a sitemap `changefreq`) // instead — a case no metric exposes. When the two disagree, the verdicts win and the divisor // is what is wrong, so say that rather than let a config gap read as a fleet failure. - const cacheServed = sumCount( - serves.filter((x) => x.method === 'hit' || x.method === 'swr' || x.method === 'peer-rescue') - ); + const cacheServed = sumCount(serves.filter((x) => isCacheServed(x.method))); const pastDue = sumCount(serves.filter((x) => x.method === 'swr' || x.method === 'stale')); const contradicted = normalizable && Number.isFinite(ratioP95) && ratioP95 > 1 && cacheServed > 0 && pastDue / cacheServed < 0.01; @@ -643,8 +679,12 @@ const FAMILIES = [ }, { key: 'invalidated', - label: 'Invalidated', - hint: 'a bulk invalidation cost the serve', + label: 'Invalidation', + // One family for both halves, because they are one population — the pages a bulk + // invalidation touched — split by what happened next: refused (`invalidated`, proxied to the + // origin) or rescued (`verified`, served from cache on the probe's evidence). Read against + // each other they size what per-page verification is buying; apart, neither means much. + hint: 'a bulk invalidation touched the serve — refused, or rescued on evidence', }, { key: 'not-cacheable', @@ -669,6 +709,13 @@ const NOT_HIT = { 'the local body failed and the residency owner’s copy answered it — still a cache serve', ], 'invalidated': ['invalidated', 'a bulk invalidation demoted a page that would otherwise have served'], + // Deliberately its own verdict upstream and never folded into `hit`: the page is as old as it + // ever was and is being served on EVIDENCE (the probe re-confirmed its price/availability + // claims after the epoch), not on age. Still a cache serve — it counts toward offload. + 'verified': [ + 'invalidated', + 'an invalidation would have refused it; the change probe proved its claims current — still a cache serve', + ], 'skip': ['not-cacheable', 'the cache was deliberately not consulted (renderNow / Cache-Control)'], 'bypass': ['not-cacheable', 'not a cacheable request at all (non-GET/HEAD)'], }; @@ -902,9 +949,10 @@ function notFreshHit(data, { serves, filter }) { 'counted in the status panel below.', ]), el('p', { cls: 'muted chart-note' }, [ - 'One row per freshness verdict, because they are four different problems: coverage is fixed in the ', + 'One row per freshness verdict, because they are different problems: coverage is fixed in the ', 'corpus (discovery, sitemaps), cadence by render capacity or a longer interval, integrity is blob ', - 'health and never a caching question, and the last two are working as configured. ', + 'health and never a caching question, invalidation is a bulk invalidation doing its job — with ', + '`verified` the part per-page verification bought back — and not-cacheable is working as configured. ', '“origin median” is what that verdict typically costs at the origin, with its p95 in the tooltip — and ', '“origin answered” is what came back: `absent` (404/410) is a page nobody has, `server-error` and ', '`connect-fail` are the origin in trouble, and only `served` is a page we could have had cached.', @@ -1059,6 +1107,140 @@ function statusCodes(data, filter) { }); } +// ---- what the origin actually saw ---------------------------------------------- +// +// The gross offload tile is a serve-side number, and the serve path is not the only thing here +// that talks to the origin. The arithmetic is `originLoad` in charts.js (the overview reads the +// same figure); this is the panel that shows its terms. + +/** What the origin saw, by cause, over time — the panel behind the net offload tile. */ +const LOAD_ROWS = [ + { + key: 'proxied', + label: 'proxied serves', + color: 'var(--warn)', + means: 'crawler requests forwarded live — the only term the gross figure counts', + }, + { + key: 'renders', + label: 'renders', + color: SERIES[0], + means: 'page loads by the render fleet, one per posted result — the document only', + }, + { + key: 'probes', + label: 'change probes', + color: SERIES[2], + means: 'origin calls by the change probe, failures included', + }, + { key: 'sitemaps', label: 'sitemap fetches', color: SERIES[3], means: 'sitemaps read by refresh runs' }, +]; + +function originSeen(data, { load, filter }) { + const stacks = originLoadBuckets(data); + const present = LOAD_ROWS.filter((row) => load[row.key] > 0); + const keys = present.map((row) => row.key); + const colorOf = (key) => LOAD_ROWS.find((row) => row.key === key)?.color ?? 'var(--fg-4)'; + + return card('What the origin actually saw', { + head: [ + spacer(), + allBotsTag(filter, 'renders, probes and sitemap fetches are not for any one crawler'), + legend(present.map((row) => ({ label: row.label, color: row.color }))), + ], + body: !load.total + ? [ + el('div', { cls: 'note ok' }, [ + 'Nothing in this window reached the origin — no proxied serve, no render, no probe, no sitemap ', + 'fetch. Offload is 100% gross and net alike.', + ]), + followUpNote(load), + ] + : [ + el('div', { cls: 'stat-grid tight' }, [ + stat('Crawlers asked for', fmtCount(load.arrived), 'requests at ingress, all bots'), + stat( + 'The origin answered', + fmtCount(load.total), + `${pct(load.total, load.arrived)} of that — ${fmtNet(load.net)} net offload`, + { warn: Number.isFinite(load.net) && load.net < 0.5 } + ), + stat( + 'Gross offload counts', + fmtCount(load.proxied), + `${fmtCount(load.total - load.proxied)} more came from this system` + ), + // The unmeasured term gets a tile so it sits in the same row as the measured ones, at + // the same size — not a footnote under a number that looks complete without it. + stat( + 'Crawler follow-up requests', + 'not measured', + `${fmtCount(load.handed)} pages handed to crawlers · what each fetched next bypasses this plugin` + ), + ]), + stackedBars(data, keys, stacks, colorOf, { format: fmtCount }), + barList( + present.map((row) => ({ + label: row.label, + value: load[row.key], + color: row.color, + sub: pct(load[row.key], load.total), + title: `${row.label}: ${num(load[row.key])} — ${row.means}`, + })), + { format: fmtCount } + ), + el('p', { cls: 'muted chart-note' }, [ + 'Every request the origin answered because this deployment exists, against what crawlers asked ', + 'for. Gross offload counts only the first bar; net offload subtracts all of them. A render is ', + 'counted as ONE request — the document; the page’s own scripts and stylesheets reach the origin ', + 'too if the CDN does not cache them for the renderer, which nothing here can see. A probe is one ', + 'small endpoint call, not a page render — cheaper per request than the others, but a request.', + load.lumpy + ? ' Probe and sitemap counts land where a PASS FINISHED, not where the requests happened: over ' + + 'a range shorter than a sweep this is either none of a running pass or all of one that just ' + + 'ended. Quote the 24h figure.' + : '', + ' Not counted: requests the CDN answered from its own cache, and renders that never posted a result.', + ]), + followUpNote(load), + Number.isFinite(load.net) && + load.net < 0 && + el('div', { cls: 'note warn' }, [ + el('strong', { text: 'The origin saw more requests than the crawlers made. ' }), + 'Net offload is negative: the render and probe cadence is generating more origin load than ', + 'the bot traffic it is standing in for. That is the expected shape of a fresh deployment ', + 'backfilling its corpus, and of a corpus much larger than what crawlers actually walk — the ', + 'levers are the render interval, the demand ladder’s floors, the discovery gate, and the ', + 'probe rate. Read it over 24h before acting on it.', + ]), + ], + }); +} + +/** + * The fifth term, spelled out. Shown whenever a page was handed to a crawler at all — including on + * the "nothing reached the origin" exit, where a net offload of 100% is precisely the claim this + * term qualifies. + */ +function followUpNote(load) { + if (!load.handed) return null; + return el('div', { cls: 'note' }, [ + el('strong', { + text: `${num(load.handed)} pages were handed to crawlers, and what each crawler fetched next is not counted. `, + }), + 'A rendering crawler (Googlebot, Bingbot, Applebot) loads the page’s scripts and then makes the XHR/fetch ', + 'calls the page makes — inventory, pricing, personalisation — and those are the calls least likely to be ', + 'cacheable; an image crawler fetches the images; any crawler may follow a linked resource. None of it ', + 'passes through this plugin (the CDN sends a crawler’s subrequests straight to the origin), so nothing ', + 'here can count it, and the net figure above is stated BEFORE it rather than with a guessed ', + 'requests-per-page multiplied in — for every crawler, not a guessed subset of them. The render fleet can ', + 'measure the per-page factor (it loads the same pages and already classifies every same-origin response ', + 'as cacheable or not); stored on the cached page and applied at serve time by what the registry says a ', + 'crawler fetches, this becomes a counted term. A snapshot served with its scripts stripped hydrates ', + 'nothing, which makes even the rendering crawlers’ share an upper bound.', + ]); +} + /** What a non-cache serve costs: why the origin was consulted, how slowly it answered, and with what. */ function originFetch(data, filter) { const fetches = pick(data, 'origin_fetch'); @@ -1114,7 +1296,7 @@ function routes(ctx, data, cadences, filter) { const route = s.path ?? 'unrouted'; const entry = byRoute.get(route) ?? { total: 0, cache: 0, miss: 0, aging: 0, integrity: 0 }; entry.total += s.count; - if (s.method === 'hit' || s.method === 'swr' || s.method === 'peer-rescue') entry.cache += s.count; + if (isCacheServed(s.method)) entry.cache += s.count; if (s.method === 'miss') entry.miss += s.count; if (s.method === 'swr' || s.method === 'stale') entry.aging += s.count; if (s.method === 'blob-missing' || s.method === 'blob-timeout' || s.method === 'peer-rescue') diff --git a/packages/console/src/util/aggregate.js b/packages/console/src/util/aggregate.js index 3f83accb..69102a1b 100644 --- a/packages/console/src/util/aggregate.js +++ b/packages/console/src/util/aggregate.js @@ -1098,9 +1098,23 @@ export function mergeChangeProbe(results) { ownerScopeNote: bodies[0].b.ownerScopeNote ?? null, rules, rulesDiverge: new Set(ruleJson).size > 1, + // Plugin v0.62.0 publishes the probe's state to a node-local row every worker can read, + // and says when that row could NOT be read: `stateAvailable: false` is "this answer is not + // trustworthy", which the old per-worker shape could not express (it reported `running: + // false, lastRun: null` — indistinguishable from the probe being off). Named per node, + // because a node whose row is unreadable contributes zeros to every sum above exactly the + // way an unswept node does, and the UI has to say which it is. An older plugin that has no + // such field is not flagged: absent is "not reported", not "unavailable". + stateUnavailableOn: bodies.filter((r) => r.b.stateAvailable === false).map((r) => r.hostname), sweep: { running: bodies.some((r) => r.b.sweep?.running), runningOn: bodies.filter((r) => r.b.sweep?.running).map((r) => r.hostname), + // Live progress of the passes in flight, per node — a sweep runs for hours, and "running" + // alone cannot tell a pass that is moving from one whose worker died (the plugin's own + // heartbeat staleness handles the latter; this is the number an operator watches move). + progress: bodies + .filter((r) => r.b.sweep?.running && r.b.sweep?.progress) + .map((r) => ({ hostname: r.hostname, ...r.b.sweep.progress })), armedInterval: bodies.find((r) => Number.isFinite(r.b.sweep?.armedInterval))?.b.sweep.armedInterval ?? null, sweptNodes: sweeps.length, unsweptNodes: bodies.filter((r) => !r.b.sweep?.lastRun?.startedAt).map((r) => r.hostname), @@ -1157,7 +1171,10 @@ export function mergeChangeProbe(results) { hostname: r.hostname, enabled: r.b.enabled ?? null, dryRun: r.b.dryRun ?? null, + // null on a plugin that predates the shared state row; false is the finding. + stateAvailable: typeof r.b.stateAvailable === 'boolean' ? r.b.stateAvailable : null, sweepRunning: !!r.b.sweep?.running, + sweepProgress: r.b.sweep?.running ? (r.b.sweep?.progress ?? null) : null, canaryRunning: !!r.b.canary?.running, sweepFinishedAt: msOf(r.b.sweep?.lastRun?.finishedAt) || null, canaryFinishedAt: msOf(r.b.canary?.lastRun?.finishedAt) || null, diff --git a/packages/console/test/adminAssets.test.js b/packages/console/test/adminAssets.test.js index 9c86b317..0ca2a3fd 100644 --- a/packages/console/test/adminAssets.test.js +++ b/packages/console/test/adminAssets.test.js @@ -257,12 +257,13 @@ test('a metric the plugin emits is charted by the console, or waived with a reas // A series may legitimately have no panel — but it has to be a decision, written down here, // not an oversight nobody noticed. + // `invalidation_error` and `invalidation_reenqueue` were waived here until console v0.12.0 — the + // Invalidations view now reads both, beside `page_verification` and the `verified` serve status + // they belong with. This test is what said the console was blind to `page_verification`. const NOT_CHARTED = new Map([ ['prerender_ops.serve_error', 'blob-fault counter; the serve-side view of it is bot_serve blob-* on Traffic'], ['prerender_ops.unrouted', 'has its own endpoint and panel (the unrouted report), not the analytics window'], ['prerender_ops.config_warnings', 'the Config view reads the warnings themselves, which say more than a count'], - ['prerender_ops.invalidation_error', 'the Invalidations view reads the rows; a failure count adds nothing'], - ['prerender_ops.invalidation_reenqueue', 'demand-driven heal is off by default and has no panel yet'], ]); const client = [...clientSources.values()].join('\n'); diff --git a/packages/console/test/aggregate.test.js b/packages/console/test/aggregate.test.js index 90c21a58..e39df3fe 100644 --- a/packages/console/test/aggregate.test.js +++ b/packages/console/test/aggregate.test.js @@ -1107,3 +1107,47 @@ test('discovery purge: a self-stopped pass is reported with the samples that exp assert.equal(body.errorSamples[0].hostname, 'b.example.com:9926'); assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').abortedOnErrors, true); }); + +// Plugin v0.62.0 moved the probe's state into a node-local row every worker can read, and made +// the endpoint say when that row could NOT be read. The merge has to carry that through: a node +// whose state is unreadable contributes zeros to every sum exactly the way an unswept node does, +// and only a name separates "idle" from "unknown". +test('change probe: a node whose state row could not be read is NAMED, not read as idle', () => { + const { body } = mergerFor('change-probe')([ + ok('a', probeBody('a', { stateAvailable: true })), + ok('b', probeBody('b', { stateAvailable: false, sweep: { running: false, armedInterval: null, lastRun: null } })), + ]); + assert.deepEqual(body.stateUnavailableOn, ['b.example.com:9926']); + assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').stateAvailable, false); + assert.equal(body.byNode.find((row) => row.hostname === 'a.example.com:9926').stateAvailable, true); +}); + +test('change probe: a plugin that predates the state row is not flagged as unavailable', () => { + const { body } = mergerFor('change-probe')([ok('a', probeBody('a')), ok('b', probeBody('b'))]); + assert.deepEqual(body.stateUnavailableOn, []); + assert.equal(body.byNode[0].stateAvailable, null); +}); + +test('change probe: live progress rides along per node, and only while the pass is running', () => { + const { body } = mergerFor('change-probe')([ + ok( + 'a', + probeBody('a', { + sweep: { running: true, armedInterval: 86_400_000, lastRun: null, progress: { examinedApprox: 12_000 } }, + }) + ), + // A finished pass that still carries a stale progress object must not be reported as moving. + ok( + 'b', + probeBody('b', { + sweep: { running: false, armedInterval: 86_400_000, lastRun: null, progress: { examinedApprox: 5 } }, + }) + ), + ]); + assert.deepEqual(body.sweep.progress, [{ hostname: 'a.example.com:9926', examinedApprox: 12_000 }]); + assert.deepEqual(body.sweep.runningOn, ['a.example.com:9926']); + assert.deepEqual(body.byNode.find((row) => row.hostname === 'a.example.com:9926').sweepProgress, { + examinedApprox: 12_000, + }); + assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').sweepProgress, null); +}); diff --git a/packages/console/test/invalidationsView.test.js b/packages/console/test/invalidationsView.test.js new file mode 100644 index 00000000..fb5d140a --- /dev/null +++ b/packages/console/test/invalidationsView.test.js @@ -0,0 +1,289 @@ +/** + * The Invalidations view, EXECUTED — its "what the invalidations are doing" panel above all. + * + * That panel reads three analytics series nobody read before console v0.12.0 (`page_verification`, + * `invalidation_reenqueue`, `invalidation_error`) beside the `verified` serve status, and the + * arithmetic has two rules a reader cannot verify from the screen: + * + * - `forwarded` is NEVER added to `lowered`. A forwarded heal is counted again by the owner under + * its own verdict in the same series, so under cluster scope adding the two double-counts. + * - Refused and rescued are ONE population split two ways, and the shares are stated against + * that population, not against all serves. + * + * The rest is the empty states: nothing invalidated is a steady state, not a fault; verification + * off explains a 100% refusal rate rather than letting it read as the feature failing. + */ + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { installDom, find } from './domShim.js'; + +installDom(); + +const { el } = await import('../src/admin/ui.js'); +const { load, render } = await import('../src/admin/views/invalidations.js'); + +const HOUR = 3_600_000; + +/** One analytics combo, in the shape `util/analyticsRead.js` emits (four buckets, flat). */ +const combo = (metric, path, method, type, count) => ({ + metric, + path, + method, + type, + count, + total: 0, + counts: [count / 4, count / 4, count / 4, count / 4], +}); + +const LIST = { + enabled: true, + maxScopes: 8, + knownScopes: ['all', 'route:prefix:/product/'], + invalidations: [ + { + scope: 'route:prefix:/product/', + resolvable: true, + invalidatedAt: new Date(Date.now() - HOUR).toISOString(), + reason: 'promo flip', + updatedBy: 'ops', + }, + ], +}; + +const EMPTY_LIST = { ...LIST, invalidations: [] }; + +const ANALYTICS = { + available: true, + scope: 'node', + node: 'node-a', + rangeMs: HOUR, + startMs: 0, + endMs: HOUR, + bucketMs: HOUR / 4, + bucketCount: 4, + scan: { ms: 4, scanned: 100, kept: 20, cap: 20_000 }, + series: [ + // 300 refused, 100 rescued: 75% / 25% of the population the invalidation touched. + combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300), + combo('bot_serve', 'cache', 'verified', 'googlebot', 100), + combo('bot_serve', 'cache', 'hit', 'googlebot', 5000), + // Verifications the sweep recorded, one of them failing to write. + combo('prerender_ops', 'page_verification', 'written', null, 4000), + combo('prerender_ops', 'page_verification', 'write-error', null, 3), + // Heal outcomes. 60 lowered on THIS node, 40 forwarded to owners (who count their own), the + // classic not-owner ceiling on the rest. + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'route:prefix:/product/', 60), + combo('prerender_ops', 'invalidation_reenqueue', 'forwarded', 'route:prefix:/product/', 40), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'route:prefix:/product/', 100), + combo('prerender_ops', 'invalidation_reenqueue', 'not-sooner', 'route:prefix:/product/', 20), + combo('prerender_ops', 'invalidation_reenqueue', 'forward-failed', 'route:prefix:/product/', 5), + ], +}; + +const config = (over = {}) => ({ + schema: { + children: { + invalidation: { + children: { + enabled: { kind: 'option' }, + verification: { children: { enabled: { kind: 'option' } } }, + reenqueue: { + children: { enabled: { kind: 'option' }, crossNode: { children: { enabled: { kind: 'option' } } } }, + }, + }, + }, + }, + }, + layers: [ + { path: 'invalidation.enabled', effective: true }, + { path: 'invalidation.verification.enabled', effective: over.verification ?? true }, + { path: 'invalidation.reenqueue.enabled', effective: over.reenqueue ?? true }, + { path: 'invalidation.reenqueue.crossNode.enabled', effective: over.crossNode ?? false }, + ], +}); + +function makeCtx({ list = LIST, analytics = ANALYTICS, cfg = config() } = {}) { + const views = {}; + const scratch = (id) => (views[id] ??= {}); + const calls = { gets: [], posts: [] }; + return { + calls, + scratch, + busy: false, + get data() { + return scratch('invalidations'); + }, + async get(route) { + calls.gets.push(route); + if (route === 'invalidations') return { ok: true, body: list }; + if (route === 'analytics') + return analytics ? { ok: true, body: analytics } : { ok: false, status: 500, body: {} }; + if (route === 'config') return { ok: true, body: cfg }; + return { ok: true, body: null }; + }, + async post(route, data) { + calls.posts.push({ route, data }); + return { ok: true, body: {} }; + }, + async run(fn) { + return fn(); + }, + fail() {}, + render() {}, + reload() {}, + go() {}, + }; +} + +const draw = (ctx) => el('div', null, render(ctx)); +const tile = (ctx, label) => + find(draw(ctx), (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === label); + +const ready = async (options) => { + const ctx = makeCtx(options); + await load(ctx); + return ctx; +}; + +test('the view reads the shared one-hour analytics window alongside the rows', async () => { + const ctx = await ready(); + assert.ok(ctx.calls.gets.includes('analytics')); + assert.match(draw(ctx).textContent, /What the invalidations are doing/); +}); + +test('refused and rescued are one population, and the shares are stated against it', async () => { + const ctx = await ready(); + const refused = tile(ctx, 'Refused'); + const rescued = tile(ctx, 'Rescued'); + assert.ok(refused && rescued, 'expected both tiles'); + assert.match(refused.textContent, /300/); + assert.match(refused.textContent, /75% of touched serves/); + assert.match(rescued.textContent, /100/); + assert.match(rescued.textContent, /25%/); + // 5,000 ordinary hits are NOT in the denominator: 300 of 5,400 would read "6%". + assert.doesNotMatch(refused.textContent, /\b6%/); +}); + +test('verification writes are counted, and a write fault is on the tile rather than in a log', async () => { + const ctx = await ready(); + const writes = tile(ctx, 'Verifications recorded'); + assert.match(writes.textContent, /4,000/); + assert.match(writes.textContent, /3 write-error/); + assert.ok( + find(writes, (n) => n.attributes?.class === 'value warn'), + 'a write fault should warn' + ); +}); + +test('forwarded heals are never added to lowered — the owner counts them under its own verdict', async () => { + const ctx = await ready(); + const heals = tile(ctx, 'Heals attempted'); + // 60 + 40 + 100 + 20 + 5 attempts; 60 lowered, not 100. + assert.match(heals.textContent, /225/); + assert.match(heals.textContent, /60 lowered/); + assert.match(heals.textContent, /44% not-owner/); + const text = draw(ctx).textContent; + for (const outcome of ['lowered', 'forwarded', 'not-owner', 'not-sooner', 'forward-failed']) { + assert.match(text, new RegExp(outcome)); + } +}); + +test('a not-owner majority warns only while cross-node forwarding is off — it is the lever', async () => { + const off = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'all', 10), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'all', 90), + ], + }, + }); + assert.ok( + find(tile(off, 'Heals attempted'), (n) => n.attributes?.class === 'value warn'), + 'should warn with crossNode off' + ); + + const on = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'all', 10), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'all', 90), + ], + }, + cfg: config({ crossNode: true }), + }); + assert.equal( + find(tile(on, 'Heals attempted'), (n) => n.attributes?.class === 'value warn'), + null + ); + assert.match(draw(on).textContent, /heal on · cross-node/); +}); + +test('an outcome this console has never heard of still gets a row — the list must sum to the attempts', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('prerender_ops', 'invalidation_reenqueue', 'teleported', 'all', 7)] }, + }); + const text = draw(ctx).textContent; + assert.match(text, /teleported/); + assert.match(tile(ctx, 'Heals attempted').textContent, /7/); +}); + +test('lkg-expired is the bad note: those requests failed OPEN', async () => { + const ctx = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_error', 'read-error', null, 4), + combo('prerender_ops', 'invalidation_error', 'lkg-expired', null, 2), + ], + }, + }); + const node = draw(ctx); + const bad = find(node, (n) => n.attributes?.class === 'note bad' && n.textContent.includes('epoch resolution')); + assert.ok(bad, 'expected a bad note'); + assert.match(bad.textContent, /6 epoch resolution failure/); + assert.match(bad.textContent, /failed OPEN/); +}); + +test('a read-error alone is a warning, not the open-failure alarm', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('prerender_ops', 'invalidation_error', 'read-error', null, 4)] }, + }); + const node = draw(ctx); + assert.ok(find(node, (n) => n.attributes?.class === 'note warn' && n.textContent.includes('epoch resolution'))); + assert.doesNotMatch(node.textContent, /failed OPEN/); +}); + +test('nothing invalidated and nothing touched is the steady state, and the panel says so', async () => { + const ctx = await ready({ + list: EMPTY_LIST, + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'cache', 'hit', 'googlebot', 500)] }, + }); + assert.match(draw(ctx).textContent, /Zeros here are the steady state, not a fault/); +}); + +test('every serve refused with verification OFF is explained as the setting, not the feature failing', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300)] }, + cfg: config({ verification: false }), + }); + const text = draw(ctx).textContent; + assert.match(text, /verification off/); + assert.match(text, /Every touched serve was refused because/); + // With verification ON and nothing rescued, the refused tile warns instead. + const on = await ready({ + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300)] }, + }); + assert.ok(find(tile(on, 'Refused'), (n) => n.attributes?.class === 'value warn')); +}); + +test('an analytics window that failed to load leaves the rest of the view standing', async () => { + const ctx = await ready({ analytics: null }); + const text = draw(ctx).textContent; + assert.match(text, /Active invalidations/); + assert.match(text, /Record an invalidation/); + assert.match(text, /No bot_serve rows/); +}); diff --git a/packages/console/test/overviewView.test.js b/packages/console/test/overviewView.test.js index d88fcda9..e4467d77 100644 --- a/packages/console/test/overviewView.test.js +++ b/packages/console/test/overviewView.test.js @@ -58,7 +58,7 @@ const CENSUS = { canceled: false, }; -function makeCtx({ overview = OVERVIEW, purge = { running: false } } = {}) { +function makeCtx({ overview = OVERVIEW, purge = { running: false }, analytics = ANALYTICS } = {}) { const views = {}; const scratch = (id) => (views[id] ??= {}); const calls = { posts: [] }; @@ -71,7 +71,7 @@ function makeCtx({ overview = OVERVIEW, purge = { running: false } } = {}) { }, async get(route) { if (route === 'overview') return { ok: true, body: overview }; - if (route === 'analytics') return { ok: true, body: ANALYTICS }; + if (route === 'analytics') return { ok: true, body: analytics }; if (route === 'discovery-purge') return { ok: true, body: purge }; return { ok: true, body: null }; }, @@ -323,3 +323,51 @@ test('a cluster where only some nodes spared bot-visited targets is called out', assert.match(text, /Only a\.example\.com:9926 spared bot-visited targets/); assert.match(text, /sum two different predicates/); }); + +// ---- the serve strip's offload tile ----------------------------------------------- + +/** One analytics combo, in the shape `util/analyticsRead.js` emits (four buckets, flat). */ +const combo = (metric, path, method, type, count, value) => ({ + metric, + path, + method, + type, + count, + total: 0, + counts: [count / 4, count / 4, count / 4, count / 4], + ...(value === undefined ? {} : { mean: value, median: value, p95: value, means: [value, value, value, value] }), +}); + +test('the offload tile shows the gross figure with the net one underneath, from the same arithmetic as Traffic', async () => { + // 900 of 1,000 serves from cache (90% gross); 500 renders and a 100-probe pass mean the origin + // answered 700 of 1,000 crawler requests — 30% net. The headline alone would be the flattering + // number; the subtitle is what stops it being quoted on its own. + const analytics = { + ...ANALYTICS, + startMs: 0, + endMs: 3_600_000, + bucketMs: 900_000, + bucketCount: 4, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 900), + combo('bot_serve', 'origin', 'miss', 'googlebot', 100), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 1000), + combo('render', 'outcome', 'rendered', null, 500), + combo('prerender_ops', 'probe_probed', null, null, 1, 100), + ], + }; + const ctx = await ready({ analytics }); + const tile = find( + draw(ctx), + (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === 'Origin offload' + ); + assert.ok(tile, 'expected an Origin offload tile'); + assert.match(tile.textContent, /90%/); + assert.match(tile.textContent, /30% net of renders \+ probes/); + assert.match(tile.textContent, /before crawler follow-up requests/); + // Below half on the net figure warns even though the gross one is fine. + assert.ok( + find(tile, (n) => n.attributes?.class === 'value warn'), + 'a 30% net offload should warn' + ); +}); diff --git a/packages/console/test/probeView.test.js b/packages/console/test/probeView.test.js index ea6791bb..c6da79a8 100644 --- a/packages/console/test/probeView.test.js +++ b/packages/console/test/probeView.test.js @@ -723,3 +723,102 @@ test('the local governor is reported apart from the origin one — the fixes poi assert.match(text, /local load/); assert.match(text, /event loop 180ms behind/); }); + +// ---- the shared state row (plugin v0.62.0) ------------------------------------- + +test('an unreadable state row is the loudest thing on the page — the node is unknown, not idle', async () => { + const ctx = await ready({ status: { ...STATUS, stateAvailable: false } }); + const text = draw(ctx).textContent; + assert.match(text, /state row could not be read on node-a/); + assert.match(text, /unknown, not idle/); +}); + +test('a cluster merge names the nodes whose state was unreadable, and flags them in the node table', async () => { + const status = { + ...STATUS, + node: null, + scope: 'cluster', + sources: { mode: 'merged', complete: true, configured: 2, answered: 2 }, + stateUnavailableOn: ['node-b'], + byNode: [ + { + hostname: 'node-a', + enabled: true, + dryRun: true, + stateAvailable: true, + sweepRunning: false, + canaryRunning: false, + }, + { + hostname: 'node-b', + enabled: true, + dryRun: true, + stateAvailable: false, + sweepRunning: false, + canaryRunning: false, + }, + ], + }; + const ctx = await ready({ status }); + const text = draw(ctx).textContent; + assert.match(text, /could not be read on node-b/); + assert.match(text, /state unreadable/); +}); + +test('a plugin without the state row is not accused of one it cannot read', async () => { + const ctx = await ready(); + assert.doesNotMatch(draw(ctx).textContent, /could not be read/); +}); + +test('a running sweep says how far it has got, at node scope and summed under the cluster merge', async () => { + const node = await ready({ + status: { ...STATUS, sweep: { ...STATUS.sweep, running: true, progress: { examinedApprox: 24_000 } } }, + }); + assert.match(draw(node).textContent, /sweep running · ~24,000 rows examined/); + + const cluster = await ready({ + status: { + ...STATUS, + node: null, + scope: 'cluster', + sources: { mode: 'merged', complete: true, configured: 2, answered: 2 }, + sweep: { + ...STATUS.sweep, + running: true, + runningOn: ['node-a', 'node-b'], + progress: [ + { hostname: 'node-a', examinedApprox: 10_000 }, + { hostname: 'node-b', examinedApprox: 4000 }, + ], + }, + byNode: [ + { + hostname: 'node-a', + enabled: true, + dryRun: true, + sweepRunning: true, + sweepProgress: { examinedApprox: 10_000 }, + }, + { + hostname: 'node-b', + enabled: true, + dryRun: true, + sweepRunning: true, + sweepProgress: { examinedApprox: 4000 }, + }, + ], + }, + }); + const text = draw(cluster).textContent; + // Disjoint slices, so the header sums; the node table keeps each node's own count. + assert.match(text, /~14,000 rows examined/); + assert.match(text, /sweeping · ~10,000 examined/); + assert.match(text, /sweeping · ~4,000 examined/); +}); + +test('a running sweep with no heartbeat count yet still reads as running, without a made-up number', async () => { + const ctx = await ready({ status: { ...STATUS, sweep: { ...STATUS.sweep, running: true, progress: null } } }); + const text = draw(ctx).textContent; + assert.match(text, /sweep running/); + assert.doesNotMatch(text, /rows examined/); +}); diff --git a/packages/console/test/trafficView.test.js b/packages/console/test/trafficView.test.js index 2c0df05d..892c4f28 100644 --- a/packages/console/test/trafficView.test.js +++ b/packages/console/test/trafficView.test.js @@ -28,7 +28,7 @@ installDom(); const { el } = await import('../src/admin/ui.js'); const { load, render, cadenceIndex, cadenceFor, notHitRows, originVerdict, originCostByReason, coverageSplit } = await import('../src/admin/views/traffic.js'); -const { ratioOf } = await import('../src/admin/charts.js'); +const { ratioOf, originLoad, fmtNet, isCacheServed } = await import('../src/admin/charts.js'); const HOUR = 3_600_000; const BUCKETS = 4; @@ -121,6 +121,13 @@ const ANALYTICS = { combo('response_500', 'p', 'GET', null, 60), // A served page whose age computed negative — dropped from the distribution above combo('prerender_ops', 'page_age_negative', 'googlebot', 'desktop', 3), + // What this system asked of the origin: 120 posted render results (one row each), one probe + // pass that made 60 requests (a VALUE — one emit carrying the count), one sitemap refresh + // that read 12 sitemaps. With 260 proxied serves the origin saw 452 of the 1,010 requests + // crawlers made: 74% gross offload, 55% net. + combo('render', 'outcome', 'rendered', null, 120), + combo('prerender_ops', 'probe_probed', null, null, 1, 60), + combo('prerender_ops', 'sitemap_sitemaps', null, null, 1, 12), ], }; @@ -409,6 +416,141 @@ test('the KPI strip separates a coverage miss from every other non-hit', async ( for (const family of ['Coverage', 'Cadence', 'Integrity']) assert.match(text, new RegExp(family)); }); +// ---- net offload ---------------------------------------------------------------- + +test('net offload subtracts every origin request this system made; gross subtracts only the proxied ones', () => { + const load = originLoad(ANALYTICS); + // 100 + 150 misses and 10 blob-timeouts, all answered from the origin. + assert.equal(load.proxied, 260); + // One row per posted result, so the emit COUNT is the render count … + assert.equal(load.renders, 120); + // … while the pass counters carry their count as a VALUE: one emit, 60 probes. sumCount here + // would report 1 and the origin load would be short by a whole pass. + assert.equal(load.probes, 60); + assert.equal(load.sitemaps, 12); + assert.equal(load.total, 452); + // The baseline is what ARRIVED (bot_request), not what was served. + assert.equal(load.arrived, 1010); + assert.ok(Math.abs(load.net - (1010 - 452) / 1010) < 1e-9); + // A pass counter contributed, so the figure is stamped as landing where a pass finished. + assert.equal(load.lumpy, true); + // The unmeasured term's exposure is every page handed to a crawler, not a subset of bots. + assert.equal(load.handed, 1010); +}); + +test('the KPI strip carries gross AND net, and says what the net figure leaves out', async () => { + const ctx = await ready(); + const text = textOf(ctx); + assert.match(text, /Origin offload · gross/); + assert.match(text, /74%/); + assert.match(text, /Origin offload · net/); + assert.match(text, /55%/); + // The fifth term is stated on the tile, on the panel's own tile, and in the note — never + // multiplied in as a guess. + assert.match(text, /before crawler follow-up requests/); + assert.match(text, /What the origin actually saw/); + assert.match(text, /Crawler follow-up requests/); + assert.match(text, /not measured/); + assert.match(text, /1,010 pages were handed to crawlers/); + // Every measured term is named with its count. + for (const term of ['proxied serves', 'renders', 'change probes', 'sitemap fetches']) + assert.match(text, new RegExp(term)); + // And the lumpiness caveat appears because a pass counter contributed. + assert.match(text, /land where a PASS FINISHED/); +}); + +test('a window with no crawler requests has no net offload — not a 100% one', () => { + const noBots = ANALYTICS.series.filter((s) => s.metric !== 'bot_request' && s.metric !== 'bot_serve'); + const load = originLoad({ ...ANALYTICS, series: noBots }); + assert.equal(load.arrived, 0); + assert.equal(load.net, null); + assert.equal(fmtNet(load.net), '—'); +}); + +test('serves stand in for arrivals only when the window has no bot_request rows at all', () => { + const noRequests = ANALYTICS.series.filter((s) => s.metric !== 'bot_request'); + assert.equal(originLoad({ ...ANALYTICS, series: noRequests }).arrived, 1010); +}); + +test('a fleet that out-requests its crawlers reads as a NEGATIVE offload, with the finding spelled out', async () => { + const analytics = { ...ANALYTICS, series: [...ANALYTICS.series, combo('render', 'outcome', 'rendered', null, 5000)] }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const figure = originLoad(analytics); + assert.ok(figure.net < 0); + // A signed percentage — pct() is for shares of a whole and would have nothing to divide by. + assert.match(fmtNet(figure.net), /^-\d+%$/); + const text = textOf(ctx); + assert.match(text, /The origin saw more requests than the crawlers made/); + // The gross figure is untouched by renders, which is the whole reason the net one exists. + assert.match(text, /74%/); +}); + +test('the net tile cannot narrow to one bot, and says so rather than showing a crawler’s number', async () => { + const ctx = await ready(); + ctx.data.bots = ['bingbot']; + const text = textOf(ctx); + assert.match(text, /before crawler follow-up requests · all bots/); + // Unchanged by the filter: renders, probes and sitemap fetches are not for any one crawler. + assert.match(text, /55%/); +}); + +test('nothing reaching the origin is 100% offload, and the follow-up caveat still stands', async () => { + const analytics = { + ...ANALYTICS, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 500), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 500), + ], + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /100% gross and net alike/); + assert.match(text, /500 pages were handed to crawlers/); +}); + +// ---- verified: a cache serve through an invalidation --------------------------- + +test('verified is a cache serve in the invalidation family — never "other", never outside cache-served', () => { + const rows = notHitRows([ + combo('bot_serve', 'cache', 'verified', 'googlebot', 40), + combo('bot_serve', 'origin', 'invalidated', 'googlebot', 60), + ]); + const byStatus = new Map(rows.map((row) => [row.status, row])); + // Same family: the population an invalidation touched, split into rescued and refused. + assert.equal(byStatus.get('verified').family, 'invalidated'); + assert.equal(byStatus.get('invalidated').family, 'invalidated'); + assert.deepEqual([...byStatus.get('verified').sources], [['cache', 40]]); + assert.ok(isCacheServed('verified')); + assert.ok(isCacheServed('peer-rescue')); + assert.ok(!isCacheServed('invalidated')); + assert.ok(!isCacheServed('blob-timeout')); +}); + +test('the per-route table counts a verified serve as cache-served', async () => { + const analytics = { + ...ANALYTICS, + series: [ + combo('bot_serve', 'cache', 'verified', 'googlebot', 50), + combo('bot_serve', 'cache', 'hit', 'googlebot', 50), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 100), + combo('route_serve', '/product/', 'verified', 'desktop', 50), + combo('route_serve', '/product/', 'hit', 'desktop', 50), + ], + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const row = rowFor(draw(ctx), '/product/'); + assert.ok(row, 'expected a /product/ row'); + // Columns: route, mode, serves, cache-served, … + assert.equal(row.children[3].textContent, '100%'); + // And the invalidation family is on the KPI strip with the verified share inside it. + const text = textOf(ctx); + assert.match(text, /Invalidation/); + assert.match(text, /rescued on evidence/); +}); + // ---- misses the origin cannot serve ----------------------------------------- test('originVerdict separates "nobody has this page" from "the origin is broken"', () => { From 819308f2350aac3a9bd9eedc3d3e6449dfde04cf Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 4 Sep 2026 13:48:14 -0400 Subject: [PATCH 2/3] fix(console): the crawler follow-up term is missing from BOTH sides of the offload ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The panel said omitting a rendering crawler's follow-up requests flattered the net figure. It does not, or not only: without this system every rendering crawler's page-view costs the origin the document PLUS the page's own API calls (the uncacheable ones), so the "crawlers asked for" baseline understates what the origin was spared — and a snapshot served without scripts triggers none of those calls, a saving the figure does not credit. Proxied pages, snapshots that keep their scripts, and our own renders still trigger them, a cost it does not charge. Documents-only on both sides, and the copy now says which way each case cuts; where snapshots ship without scripts the true net for rendering crawlers is HIGHER than shown. Co-Authored-By: Claude Code --- packages/console/README.md | 16 +++++--- packages/console/src/admin/charts.js | 45 ++++++++++++--------- packages/console/src/admin/views/traffic.js | 38 +++++++++-------- packages/console/test/trafficView.test.js | 4 ++ 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/packages/console/README.md b/packages/console/README.md index 5960b4e6..90d9e093 100644 --- a/packages/console/README.md +++ b/packages/console/README.md @@ -176,11 +176,17 @@ origin request (the document — the page's own subresources reach the origin on not cache them for the renderer), and the probe and sitemap counters land in the bucket where a _pass finished_, so a short range reads either none of a running sweep or all of one that just ended — quote the 24h figure. One term the plugin cannot see at all is stated rather than -omitted: whatever a crawler fetches from the origin _after_ we hand it a page — a rendering -crawler's scripts and then the page's own XHR/API calls above all — never passes through this -plugin, so the net tile says "before crawler follow-up requests" and the panel reports the -exposure (every page handed to a crawler) as a count, never multiplied by a guessed factor. The -render fleet can measure the per-page factor; when a plugin reports it, that tile becomes a number. +omitted, and it is missing from _both_ sides: the requests a page's own scripts make when a +rendering crawler runs it. Without this system every Googlebot/Bingbot/Applebot page-view costs the +origin the document _plus_ the page's XHR/API calls (the ones no CDN caches), so the "crawlers asked +for" baseline understates what the origin was spared; with it, a snapshot served without scripts +triggers none of those calls (a saving the figure does not credit), while a snapshot that keeps its +scripts, a proxied origin page, and every one of our own renders still trigger them (a cost it does +not charge). None of it passes through the plugin, so the figure is documents-only on both sides, +the net tile says "before crawler follow-up requests", and the panel reports the exposure (every +page handed to a crawler) as a count, never multiplied by a guessed factor — where snapshots are +served with scripts stripped, the true net offload for rendering crawlers is _higher_ than shown. +The render fleet can measure the per-page factor; applied to both sides, that tile becomes a number. **Invalidations gained a third panel** for the same release: what the active rows are doing. Refused serves (`invalidated` — each an origin round trip) beside rescued ones (`verified`, plugin diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index 3cae3502..8ea972f5 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -597,24 +597,31 @@ const SITEMAP_FETCHES = 'sitemap_sitemaps'; * baseline is what ARRIVED. When the window has serves but no requests (it should not — both are * gated identically — but an older plugin could), the serve total stands in. * - * A FIFTH TERM IS EXPOSED BUT NOT COUNTED — what a crawler fetches from the origin AFTER we hand - * it a page. A rendering crawler (Google's WRS, Bingbot, Applebot) loads the page's scripts and - * then makes whatever XHR/fetch calls the page makes — inventory, pricing, personalisation — and - * those are the calls least likely to be cacheable anywhere; an image crawler fetches the images; - * any crawler may follow a linked resource. None of it passes through this plugin: the CDN - * forwards the DOCUMENT request to us and sends the crawler's subrequests straight to the origin, - * so no series here can count them, and a net figure that quietly omitted them would be - * flattering in exactly the way the gross one is. What CAN be counted is the exposure: every page - * handed to a crawler — ALL of them, not a guessed subset of bots that "probably" fetch more, - * because which crawler does what is a fact about the crawler that this console should not - * assert. It is reported as that — a count of pages, never multiplied by a guessed - * requests-per-page — and every reader of this figure says "before crawler follow-up requests". - * The measured version needs the render fleet and the registry: our own headless Chrome loads the - * same pages and already runs a cache policy on every same-origin response, so "uncacheable - * same-origin subrequests per page load" is measurable there; stored on the cached page and - * multiplied at serve time by what the registry says that crawler fetches, it becomes a counted - * term. A snapshot with its scripts stripped hydrates nothing, which makes even the rendering - * crawlers' share of this an upper bound. + * A FIFTH TERM IS MISSING FROM BOTH SIDES OF THE LEDGER, and it is stated rather than guessed: + * the requests a page's OWN SCRIPTS make. A rendering crawler (Google's WRS, Bingbot, Applebot) + * fetches a page and then runs it, and the page makes its XHR/fetch calls — inventory, pricing, + * personalisation — which are exactly the calls no CDN caches. Call that k per page load. Then: + * + * without us every rendering crawler's request costs the origin 1 + k, not 1 — the baseline + * above (`bot_request`) counts documents only, so it UNDERSTATES what the origin + * was spared. + * with us a snapshot served with its scripts stripped triggers NONE of the k (a saving not + * credited above); a snapshot that keeps its scripts, and every proxied origin page, + * trigger them as before (a cost not charged above); and our own renders run the + * page too, so each render costs 1 + k, not the 1 counted above. + * + * None of these calls pass through this plugin — the CDN forwards the DOCUMENT to us and sends a + * crawler's subrequests straight to the origin — so no series here can count any of them, and the + * sign of the omission depends on facts this plugin cannot see (whether served snapshots carry + * scripts, which crawlers render). So the figure is documents-only on BOTH sides, says so, and + * reports the exposure: every page handed to a crawler — ALL of them, not a guessed subset of bots, + * because which crawler runs what is a fact about the crawler that this console should not assert. + * Where snapshots are served without scripts, the true net offload for rendering crawlers is + * HIGHER than shown. The measured version needs the render fleet and the registry: our headless + * Chrome already runs a cache policy on every same-origin response, so k ("uncacheable same-origin + * subrequests per page load") is measurable per render; stored on the cached page and applied to + * all four places above by what the registry says each crawler runs, it becomes a counted term on + * both sides. * * WHAT IS NOT COUNTED, so nobody assumes it is: crawler requests the CDN answered from its own * cache (they never reach this plugin), renders that crashed before posting a result (no row), and @@ -646,7 +653,7 @@ export function originLoad(data) { net: arrived > 0 ? (arrived - total) / arrived : null, lumpy: probes > 0 || sitemaps > 0, // The fifth term's exposure: every page handed to a crawler, cache-served or proxied alike — - // the crawler fetches what it fetches next either way. + // each one is a page whose scripts the crawler either ran against the origin or did not. handed: served, }; } diff --git a/packages/console/src/admin/views/traffic.js b/packages/console/src/admin/views/traffic.js index f173f587..2c42fbb8 100644 --- a/packages/console/src/admin/views/traffic.js +++ b/packages/console/src/admin/views/traffic.js @@ -25,10 +25,11 @@ * system asks of the origin in exchange — every render, every change probe, every sitemap fetch is * an origin request. Net offload subtracts those (`originLoad` in charts.js carries the arithmetic * and its caveats), and the panel beside the origin-fetch chart shows each term. One term it - * cannot count is stated rather than omitted: whatever a crawler fetches from the origin after we - * hand it a page (a rendering crawler's scripts and API calls above all), which never passes - * through this plugin. The net tile says "before crawler follow-up requests" and the panel reports - * the exposure — every page handed to a crawler — as a count, never multiplied by a guessed factor. + * cannot count is stated rather than omitted: the requests a page's own scripts make when a + * rendering crawler runs it, which never pass through this plugin and are missing from BOTH sides + * — the origin would have served them for every raw page-view, and does not for a snapshot served + * without scripts. The net tile says "before crawler follow-up requests" and the panel reports the + * exposure — every page handed to a crawler — as a count, never multiplied by a guessed factor. * * AND NOT EVERY MISS IS OURS. A miss whose origin fetch came back 404 or 410 is a URL that does * not exist anywhere — there is nothing for the cache to be missing, and it can never improve, @@ -1175,7 +1176,7 @@ function originSeen(data, { load, filter }) { stat( 'Crawler follow-up requests', 'not measured', - `${fmtCount(load.handed)} pages handed to crawlers · what each fetched next bypasses this plugin` + `${fmtCount(load.handed)} pages handed to crawlers · counted on neither side of the ledger` ), ]), stackedBars(data, keys, stacks, colorOf, { format: fmtCount }), @@ -1220,24 +1221,27 @@ function originSeen(data, { load, filter }) { /** * The fifth term, spelled out. Shown whenever a page was handed to a crawler at all — including on * the "nothing reached the origin" exit, where a net offload of 100% is precisely the claim this - * term qualifies. + * term qualifies (in that case, upward). */ function followUpNote(load) { if (!load.handed) return null; return el('div', { cls: 'note' }, [ el('strong', { - text: `${num(load.handed)} pages were handed to crawlers, and what each crawler fetched next is not counted. `, + text: `${num(load.handed)} pages were handed to crawlers, and the requests their scripts make are counted on neither side. `, }), - 'A rendering crawler (Googlebot, Bingbot, Applebot) loads the page’s scripts and then makes the XHR/fetch ', - 'calls the page makes — inventory, pricing, personalisation — and those are the calls least likely to be ', - 'cacheable; an image crawler fetches the images; any crawler may follow a linked resource. None of it ', - 'passes through this plugin (the CDN sends a crawler’s subrequests straight to the origin), so nothing ', - 'here can count it, and the net figure above is stated BEFORE it rather than with a guessed ', - 'requests-per-page multiplied in — for every crawler, not a guessed subset of them. The render fleet can ', - 'measure the per-page factor (it loads the same pages and already classifies every same-origin response ', - 'as cacheable or not); stored on the cached page and applied at serve time by what the registry says a ', - 'crawler fetches, this becomes a counted term. A snapshot served with its scripts stripped hydrates ', - 'nothing, which makes even the rendering crawlers’ share an upper bound.', + 'A rendering crawler (Googlebot, Bingbot, Applebot) fetches a page and then runs it, and the page makes ', + 'its own XHR/fetch calls to the origin — inventory, pricing, personalisation — which are exactly the ', + 'calls no CDN caches. Without this system every such crawl costs the origin the document PLUS those ', + 'calls, so the “crawlers asked for” baseline above understates what the origin was spared. With it, a ', + 'snapshot served without its scripts triggers none of them (a saving not credited above), a snapshot ', + 'that keeps its scripts or a proxied origin page triggers them as before (a cost not charged), and our ', + 'own renders run the page too (each render is really the document plus those calls). None of it passes ', + 'through this plugin — the CDN sends a crawler’s subrequests straight to the origin — so the figure is ', + 'documents-only on both sides and says so, for every crawler rather than a guessed subset. Where ', + 'snapshots are served with scripts stripped, the true net offload for rendering crawlers is HIGHER than ', + 'shown. The render fleet can measure the per-page factor (it loads the same pages and already classifies ', + 'every same-origin response as cacheable or not); applied to both sides by what the registry says each ', + 'crawler runs, this becomes a counted term.', ]); } diff --git a/packages/console/test/trafficView.test.js b/packages/console/test/trafficView.test.js index 892c4f28..f9110d06 100644 --- a/packages/console/test/trafficView.test.js +++ b/packages/console/test/trafficView.test.js @@ -452,6 +452,10 @@ test('the KPI strip carries gross AND net, and says what the net figure leaves o assert.match(text, /Crawler follow-up requests/); assert.match(text, /not measured/); assert.match(text, /1,010 pages were handed to crawlers/); + // The omitted term is stated as two-sided: a saving where snapshots carry no scripts, a cost + // where they do — never as "the number would only be lower". + assert.match(text, /counted on neither side/); + assert.match(text, /true net offload for rendering crawlers is HIGHER than shown/); // Every measured term is named with its count. for (const term of ['proxied serves', 'renders', 'change probes', 'sitemap fetches']) assert.match(text, new RegExp(term)); From edd9af164a66f143c64601ea6b4f38fcdf424f36 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 4 Sep 2026 13:51:41 -0400 Subject: [PATCH 3/3] fix(console): net offload through ratioOf, like every other division in charts.js (review) Co-Authored-By: Claude Code --- packages/console/src/admin/charts.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index 8ea972f5..b3085e0f 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -649,8 +649,9 @@ export function originLoad(data) { sitemaps, total, arrived, - // Null, never 0 or 100%: an empty window has no offload to report. - net: arrived > 0 ? (arrived - total) / arrived : null, + // Null, never 0 or 100%: an empty window has no offload to report. Through ratioOf like every + // other division in this module, so the zero-denominator guard cannot be forgotten here either. + net: ratioOf(arrived - total, arrived), lumpy: probes > 0 || sitemaps > 0, // The fifth term's exposure: every page handed to a crawler, cache-served or proxied alike — // each one is a page whose scripts the crawler either ran against the origin or did not.