diff --git a/package-lock.json b/package-lock.json index e724345c..9b93a599 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8556,7 +8556,7 @@ }, "packages/console": { "name": "@harperfast/prerender-console", - "version": "0.11.0", + "version": "0.12.0", "license": "Apache-2.0", "dependencies": { "undici": "^7.18.2" diff --git a/packages/console/README.md b/packages/console/README.md index cd636485..90d9e093 100644 --- a/packages/console/README.md +++ b/packages/console/README.md @@ -162,6 +162,43 @@ not have — the discovery gate, and a client-side bot filter), **Sitemaps**, ** **Queue** (render/claim health and the backlog), **Nodes**, **Invalidations** (preview-first record/clear), **Change probe**, **URL explainer**, **Metrics** (the live catalog), **Config**. +**Offload is stated twice, gross and net** (console v0.12.0). The gross figure — crawler requests +not proxied to the origin live — is what every serve-side panel is built from, and it is +flattering: it counts what the origin was spared and none of what this system asks of the origin +in exchange. Every render is the headless fleet loading a page from the origin; every change probe +is an origin call; every sitemap refresh fetches the sitemap. A deployment rendering its whole +corpus on a short cadence for a trickle of bot traffic can post a 95% gross offload while sending +the origin _more_ requests than the crawlers would have. Net offload is +`1 − (proxied + renders + probes + sitemap fetches) ÷ crawler requests arrived`, every term from a +series already in the one analytics scan, and the panel beside the origin-fetch chart shows each +term over time. Two caveats are written on the panel rather than assumed: a render counts as one +origin request (the document — the page's own subresources reach the origin only if the CDN does +not cache them for the renderer), and the probe and sitemap counters land in the bucket where a +_pass finished_, so a short range reads either none of a running sweep or all of one that just +ended — quote the 24h figure. One term the plugin cannot see at all is stated rather than +omitted, and it is missing from _both_ sides: the requests a page's own scripts make when a +rendering crawler runs it. Without this system every Googlebot/Bingbot/Applebot page-view costs the +origin the document _plus_ the page's XHR/API calls (the ones no CDN caches), so the "crawlers asked +for" baseline understates what the origin was spared; with it, a snapshot served without scripts +triggers none of those calls (a saving the figure does not credit), while a snapshot that keeps its +scripts, a proxied origin page, and every one of our own renders still trigger them (a cost it does +not charge). None of it passes through the plugin, so the figure is documents-only on both sides, +the net tile says "before crawler follow-up requests", and the panel reports the exposure (every +page handed to a crawler) as a count, never multiplied by a guessed factor — where snapshots are +served with scripts stripped, the true net offload for rendering crawlers is _higher_ than shown. +The render fleet can measure the per-page factor; applied to both sides, that tile becomes a number. + +**Invalidations gained a third panel** for the same release: what the active rows are doing. +Refused serves (`invalidated` — each an origin round trip) beside rescued ones (`verified`, plugin +v0.63.0 — a page the change probe has _proved_ current since the epoch, served from cache through +the invalidation), the verifications the sweep recorded, and every outcome of the demand-driven +heal including v0.64.0's cross-node `forwarded` / `forward-failed`. `forwarded` is deliberately +never added to `lowered`: the owner counts a forwarded heal under its own verdict in the same +series, so adding the two double-counts under cluster scope. The `verified` status is also a +cache serve everywhere else in the console — the freshness chart, the per-route table, the +staleness sums — and sits in one "Invalidation" family with `invalidated` on the non-hit strip, +because the two are one population split into rescued and refused. + **Change probe is new** (plugin v0.53.0+), and it is the one freshness surface that does not measure pages against a cadence. It reads what the probe is actually detecting — the change rate against the probes that HAD a baseline, not against every probe, because a pass that is mostly seeding has @@ -182,6 +219,13 @@ flagged when a settled deployment skips most of them, which means `reprobeAfter` `sweepInterval`), and registry rows that could not be decoded, which is a storage-layer escalation rather than anything a setting here reaches. +Plugin v0.62.0 moved the probe's state into a node-local row every worker can read (before it, 15 +of 16 workers answered "not running, never ran" — indistinguishable from the probe being off), and +the endpoint now says when that row could _not_ be read. The console prints that as the loudest +note on the page — the node is unknown, not idle — and shows a running sweep's heartbeat count +("~24,000 rows examined") on the header pill, the sweep card and the per-node table, summed across +the nodes running because each walks its own slice. + Plugin v0.58.0's `pageCheck` adds the one counter on that card that is not about the origin changing: **page mismatches** — cached pages that disagree with the origin on a field they claim. That is the class the signature comparison is structurally blind to (a value that changed and diff --git a/packages/console/package.json b/packages/console/package.json index 4f994f78..62f62687 100644 --- a/packages/console/package.json +++ b/packages/console/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-console", - "version": "0.11.0", + "version": "0.12.0", "type": "module", "description": "Standalone Harper component serving the prerender management console UI, proxying to a prerender deployment's /prerender_admin API", "license": "Apache-2.0", diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index 7da1d8d0..b3085e0f 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -40,6 +40,10 @@ export const CACHE_STATUS_COLORS = { 'hit': OK, 'swr': INFO, 'peer-rescue': '#8ff1cd', + // A cache serve an invalidation would have refused, let through on the probe's evidence + // (plugin v0.63.0). Coloured beside `invalidated` rather than beside `hit`: the two are the + // same population — pages a bulk invalidation touched — split into rescued and refused. + 'verified': '#c9a7ff', 'miss': WARN, 'stale': PINK, 'invalidated': PURPLE, @@ -49,6 +53,19 @@ export const CACHE_STATUS_COLORS = { 'blob-timeout': BAD, }; +/** + * The freshness verdicts that are CACHE SERVES — the bytes came from a stored snapshot. This is + * the one enumeration every "cache-served" sum in the console must share, because a verdict + * missing from it does not read as missing: it reads as a smaller hit rate. `verified` sat outside + * every such sum for one plugin release and the cache-served share simply looked lower. + * + * `hit`, `swr` and `verified` are the page itself; `peer-rescue` is the owner's copy of it. What + * is NOT here: `miss`/`stale` (origin), `blob-*` (the local body failed AND no rescue landed, so + * the request went to origin), `invalidated` (refused), `skip`/`bypass` (never consulted). + */ +export const CACHE_SERVED = new Set(['hit', 'swr', 'verified', 'peer-rescue']); +export const isCacheServed = (status) => CACHE_SERVED.has(status); + /** Where the bytes came from (bot_serve.path). `origin` is the one offload counts against. */ export const SOURCE_COLORS = { cache: OK, rendered: INFO, origin: WARN }; @@ -534,3 +551,166 @@ export const emptyNote = (what, data) => el('code', { text: 'analytics.enabled' }), ' gates recording). Analytics rows are node-local; a cluster view sums every node’s own slice.', ]); + +// ---- origin load: the net offload arithmetic --------------------------------- +// +// Shared by Traffic (which shows every term) and the Overview (which shows the figure), so the two +// can never disagree about what "net" means. + +// Series names as constants: the route-contract scanner in adminAssets.test.js reads a quoted +// name inside a Map lookup or a comparison as a fetch of a route by that name. +const RENDER_OUTCOME = 'outcome'; +const PROBE_REQUESTS = 'probe_probed'; +const SITEMAP_FETCHES = 'sitemap_sitemaps'; + +/** + * Every request the origin answered because this deployment exists, over the window, beside the + * requests crawlers made — the two sides of the net offload figure. + * + * GROSS OFFLOAD IS FLATTERING. It is the share of crawler requests that were not proxied live + * (`bot_serve` source ≠ origin), and it counts what the origin was spared and none of what this + * system asks of the origin in exchange. A deployment rendering its whole corpus on a short + * cadence for a trickle of bot traffic can post a 95% gross offload while sending the origin MORE + * requests than the crawlers would have. Net offload is + * `1 − (proxied + renders + probes + sitemap fetches) ÷ crawler requests arrived`. + * + * FOUR TERMS, each from a series already in the scan: + * + * proxied `bot_serve` source = origin. A crawler request forwarded live: a miss, a page past + * its SWR window, a blob fault nobody rescued, an invalidated page, a skip, a bypass. + * renders `render` series `outcome` — EXACTLY one row per posted result (metrics.js), so its + * count is the number of page loads the headless fleet performed against the origin, + * whatever became of each (rendered, suppressed, failed, transient…). Each is counted as + * ONE origin request: the document. A page load also pulls the page's own scripts and + * stylesheets, which reach the origin only if the CDN does not cache them for the + * renderer — a deployment fact no metric here can see, so it is stated, not estimated. + * probes `probe_probed` — one origin call per attempt, failures included (a refused probe was + * still a request). Sweep and canary both emit it. A probe hits a small endpoint, not a + * page render, so it is cheaper than the other three; it is still a request. + * sitemaps `sitemap_sitemaps` — one fetch per sitemap a refresh run processed. + * + * `renders` is read with sumCount (one emit = one result); the two pass counters with sumValues + * (one emit per pass carrying the pass's count — sumCount there would count passes). + * + * THE DENOMINATOR IS `bot_request`, not `bot_serve`: without this system the CDN forwards every + * crawler request to the origin, including the few that never reach a serve outcome here, so the + * baseline is what ARRIVED. When the window has serves but no requests (it should not — both are + * gated identically — but an older plugin could), the serve total stands in. + * + * A FIFTH TERM IS MISSING FROM BOTH SIDES OF THE LEDGER, and it is stated rather than guessed: + * the requests a page's OWN SCRIPTS make. A rendering crawler (Google's WRS, Bingbot, Applebot) + * fetches a page and then runs it, and the page makes its XHR/fetch calls — inventory, pricing, + * personalisation — which are exactly the calls no CDN caches. Call that k per page load. Then: + * + * without us every rendering crawler's request costs the origin 1 + k, not 1 — the baseline + * above (`bot_request`) counts documents only, so it UNDERSTATES what the origin + * was spared. + * with us a snapshot served with its scripts stripped triggers NONE of the k (a saving not + * credited above); a snapshot that keeps its scripts, and every proxied origin page, + * trigger them as before (a cost not charged above); and our own renders run the + * page too, so each render costs 1 + k, not the 1 counted above. + * + * None of these calls pass through this plugin — the CDN forwards the DOCUMENT to us and sends a + * crawler's subrequests straight to the origin — so no series here can count any of them, and the + * sign of the omission depends on facts this plugin cannot see (whether served snapshots carry + * scripts, which crawlers render). So the figure is documents-only on BOTH sides, says so, and + * reports the exposure: every page handed to a crawler — ALL of them, not a guessed subset of bots, + * because which crawler runs what is a fact about the crawler that this console should not assert. + * Where snapshots are served without scripts, the true net offload for rendering crawlers is + * HIGHER than shown. The measured version needs the render fleet and the registry: our headless + * Chrome already runs a cache policy on every same-origin response, so k ("uncacheable same-origin + * subrequests per page load") is measurable per render; stored on the cached page and applied to + * all four places above by what the registry says each crawler runs, it becomes a counted term on + * both sides. + * + * WHAT IS NOT COUNTED, so nobody assumes it is: crawler requests the CDN answered from its own + * cache (they never reach this plugin), renders that crashed before posting a result (no row), and + * cluster-internal traffic — peer rescue and forwarded heals are node-to-node, never origin. + * + * `lumpy` is true when either pass counter contributed: those land in the bucket where the pass + * FINISHED, so over a range shorter than a pass the term is either absent or all of it. Over a + * 24h range it is right to within one pass. + */ +export function originLoad(data) { + const serves = pick(data, 'bot_serve'); + const served = sumCount(serves); + const proxied = sumCount(serves.filter((s) => s.path === 'origin')); + const renders = sumCount(pick(data, 'render', (s) => s.path === RENDER_OUTCOME)); + const probes = sumValues(pick(data, 'prerender_ops', (s) => s.path === PROBE_REQUESTS)); + const sitemaps = sumValues(pick(data, 'prerender_ops', (s) => s.path === SITEMAP_FETCHES)); + const requests = sumCount(pick(data, 'bot_request')); + const arrived = requests > 0 ? requests : served; + const total = proxied + renders + probes + sitemaps; + + return { + proxied, + renders, + probes, + sitemaps, + total, + arrived, + // Null, never 0 or 100%: an empty window has no offload to report. Through ratioOf like every + // other division in this module, so the zero-denominator guard cannot be forgotten here either. + net: ratioOf(arrived - total, arrived), + lumpy: probes > 0 || sitemaps > 0, + // The fifth term's exposure: every page handed to a crawler, cache-served or proxied alike — + // each one is a page whose scripts the crawler either ran against the origin or did not. + handed: served, + }; +} + +/** + * Per-bucket origin load by cause, for a stacked chart: stackBy's inner loop applied per TERM + * rather than over one metric's dimension, because the four terms come from three metrics and two + * value semantics. The proxied and render terms are per-request counts; the two pass counters are + * VALUES (one emit carrying a count), so their per-bucket form is mean × count, as in stackBy's + * values mode. + */ +export function originLoadBuckets(data) { + const bucketsOf = (combos, values) => { + const out = new Array(data.bucketCount).fill(0); + for (const combo of combos) { + for (let i = 0; i < combo.counts.length && i < out.length; i++) { + if (!values) out[i] += combo.counts[i]; + else if (Number.isFinite(combo.means?.[i]) && combo.counts[i] > 0) out[i] += combo.means[i] * combo.counts[i]; + } + } + return out; + }; + return new Map([ + [ + 'proxied', + bucketsOf( + pick(data, 'bot_serve', (s) => s.path === 'origin'), + false + ), + ], + [ + 'renders', + bucketsOf( + pick(data, 'render', (s) => s.path === RENDER_OUTCOME), + false + ), + ], + [ + 'probes', + bucketsOf( + pick(data, 'prerender_ops', (s) => s.path === PROBE_REQUESTS), + true + ), + ], + [ + 'sitemaps', + bucketsOf( + pick(data, 'prerender_ops', (s) => s.path === SITEMAP_FETCHES), + true + ), + ], + ]); +} + +/** + * A net offload as a percentage that may be NEGATIVE. `pct()` is built for shares of a whole; + * this is a signed ratio of 1, and a minus sign here is the finding. + */ +export const fmtNet = (net) => (Number.isFinite(net) ? `${Math.round(net * 100)}%` : '—'); diff --git a/packages/console/src/admin/views/invalidations.js b/packages/console/src/admin/views/invalidations.js index 81634a20..2b29c11e 100644 --- a/packages/console/src/admin/views/invalidations.js +++ b/packages/console/src/admin/views/invalidations.js @@ -13,16 +13,45 @@ * AN UNRESOLVABLE ROW IS THE LOUDEST THING ON THE PAGE. A row whose scope no longer names * a configured route looks applied and matches nothing — worse than no row — so it gets * the bad pill, not a footnote. + * + * WHAT AN INVALIDATION IS DOING is the third panel, and it is the one that needs the analytics + * window. A row on the table says what is invalidated; it says nothing about what that costs + * (every refused serve is an origin round trip) or what the two mechanisms built to cut that + * cost are buying back — per-page verification (plugin v0.63.0: a page the probe has PROVED + * current is served through the invalidation, `bot_serve` cacheStatus `verified`) and the + * demand-driven heal (a crawled invalidated page is pulled forward in the queue, with v0.64.0 + * forwarding the heal to the key's owner instead of refusing it as `not-owner`). Those are + * counters, they live in the analytics scan, and this is the view an operator is on when they + * ask "is it working". The read is the same one-hour window the overview already holds, so on a + * warm cache it costs no scan at all. */ -import { ago, card, el, ICONS, kv, mono, muted, num, pill, spacer, table } from '../ui.js'; -import { appliedNote, editTray, loadConfig, settingsCard } from './_configEdit.js'; +import { ago, card, el, ICONS, kv, mono, muted, num, pct, pill, spacer, stat, table } from '../ui.js'; +import { + barList, + emptyNote, + fmtCount, + pick, + scanFooter, + scopeLabel, + SERIES, + sumCount, + windowEmpty, +} from '../charts.js'; +import { appliedNote, configState, editTray, loadConfig, optionIndex, settingsCard } from './_configEdit.js'; export const meta = { id: 'invalidations', label: 'Invalidations', crumb: 'invalidations', icon: ICONS.invalidations }; export async function load(ctx) { - const [res] = await Promise.all([ctx.get('invalidations'), loadConfig(ctx)]); + const [res, analyticsRes] = await Promise.all([ + ctx.get('invalidations'), + // The same range key Overview, Queue and Nodes use, so this is served from the worker's + // cache whenever any of them loaded inside management.analytics.cacheTtl. + ctx.get('analytics', { range: 3_600_000 }), + loadConfig(ctx), + ]); ctx.data.list = res.ok ? res.body : null; + ctx.data.analytics = analyticsRes.ok ? analyticsRes.body : null; ctx.data.error = res.ok ? null : (res.body?.error ?? `Could not load invalidations (${res.status})`); } @@ -45,12 +74,249 @@ export function render(ctx) { ' is false — they are recorded and NOT enforced. Whatever those rows were protecting against is being served.', ]), active(ctx, data), + effect(ctx, data), record(ctx, data), settings(ctx), editTray(ctx), ]; } +// ---- what the invalidations are doing --------------------------------------- + +// Series and outcome names as constants: the route-contract test in adminAssets.test.js reads a +// quoted name inside a Map lookup as a call to an admin route of that name. +const PAGE_VERIFICATION = 'page_verification'; +const REENQUEUE = 'invalidation_reenqueue'; +const INVALIDATION_ERROR = 'invalidation_error'; +const VERIFIED = 'verified'; +const INVALIDATED = 'invalidated'; +const WRITTEN = 'written'; +const LOWERED = 'lowered'; +const NOT_OWNER = 'not-owner'; +const LKG_EXPIRED = 'lkg-expired'; + +/** + * Every outcome the heal can record, with what it means — the closed set the plugin exports as + * `REENQUEUE_OUTCOMES`. Order is the order an operator should read them in: what worked, what was + * handed on, then why the rest did not, most actionable first. + * + * `forwarded` and `lowered` are deliberately never added together: a forwarded heal is counted + * again by the OWNER under its own verdict in this same series, so under cluster scope `lowered` + * already includes the forwarded heals that landed and adding `forwarded` would double-count them. + */ +const HEAL_OUTCOMES = [ + { + key: 'lowered', + label: 'lowered', + tone: 'ok', + means: 'the URL’s due time was pulled forward — it heals ahead of its cadence', + }, + { + key: 'forwarded', + label: 'forwarded', + tone: 'info', + means: 'this node did not own the key and handed the heal to the owner, which counts its own verdict here', + }, + { + key: 'not-owner', + label: 'not-owner', + tone: 'warn', + means: 'this node did not own the key and cross-node forwarding is off — the heal was dropped', + }, + { + key: 'forward-failed', + label: 'forward-failed', + tone: 'bad', + means: 'the owner could not be reached in time — a transport fault, not a refusal', + }, + { key: 'throttled', label: 'throttled', tone: 'warn', means: 'invalidation.reenqueue.maxPerMinute was spent' }, + { key: 'not-sooner', label: 'not-sooner', tone: '', means: 'already due at least as soon — nothing to pull forward' }, + { + key: 'unhealable', + label: 'unhealable', + tone: '', + means: 'already rendered after the epoch, or striking out — a re-render would not help', + }, + { key: 'leased', label: 'leased', tone: '', means: 'a render is in flight for this key right now' }, + { key: 'paused', label: 'paused', tone: '', means: 'the queue is paused on this node' }, + { + key: 'no-schedule', + label: 'no-schedule', + tone: 'bad', + means: 'a live URL with no schedule row — the terminal gap render.reconcile repairs', + }, + { key: 'no-target', label: 'no-target', tone: '', means: 'no Target for the URL — nothing to schedule' }, + { key: 'error', label: 'error', tone: 'bad', means: 'the schedule write threw' }, +]; + +/** Total per `dim` value over the given combos, as an ordered list. */ +function tally(combos, dim) { + const totals = new Map(); + for (const s of combos) totals.set(s[dim] ?? 'unknown', (totals.get(s[dim] ?? 'unknown') ?? 0) + s.count); + return totals; +} + +function effect(ctx, list) { + const data = ctx.data.analytics; + const title = data + ? `What the invalidations are doing — ${scopeLabel(data)}, last hour` + : 'What the invalidations are doing'; + const options = optionIndex(configState(ctx).payload); + const on = (path) => options.get(path)?.effective === true; + const verificationOn = on('invalidation.verification.enabled'); + const reenqueueOn = on('invalidation.reenqueue.enabled'); + const crossNodeOn = on('invalidation.reenqueue.crossNode.enabled'); + + if (!data || data.available === false || windowEmpty(data)) { + return card(title, { body: [emptyNote('bot_serve', data)] }); + } + + const serves = pick(data, 'bot_serve'); + const refused = sumCount(serves.filter((s) => s.method === INVALIDATED)); + const rescued = sumCount(serves.filter((s) => s.method === VERIFIED)); + const touched = refused + rescued; + + const writes = tally( + pick(data, 'prerender_ops', (s) => s.path === PAGE_VERIFICATION), + 'method' + ); + const written = writes.get(WRITTEN) ?? 0; + const writeFaults = [...writes.entries()].filter(([outcome]) => outcome !== WRITTEN); + + const heals = tally( + pick(data, 'prerender_ops', (s) => s.path === REENQUEUE), + 'method' + ); + const healTotal = [...heals.values()].reduce((acc, n) => acc + n, 0); + const healRows = HEAL_OUTCOMES.filter((o) => heals.has(o.key)).map((o) => ({ ...o, count: heals.get(o.key) })); + // An outcome this console has no entry for still gets a row, uncoloured, rather than vanishing + // from a list whose whole point is that it sums to the attempts. + for (const [outcome, count] of heals) { + if (!HEAL_OUTCOMES.some((o) => o.key === outcome)) { + healRows.push({ + key: outcome, + label: outcome, + tone: '', + means: 'an outcome this console does not know about', + count, + }); + } + } + const notOwner = heals.get(NOT_OWNER) ?? 0; + + const errors = tally( + pick(data, 'prerender_ops', (s) => s.path === INVALIDATION_ERROR), + 'method' + ); + const lkgExpired = errors.get(LKG_EXPIRED) ?? 0; + const errorTotal = [...errors.values()].reduce((acc, n) => acc + n, 0); + + const nothingActive = !(list.invalidations?.length > 0); + + return card(title, { + head: [ + spacer(), + verificationOn ? pill('verification on', 'ok') : pill('verification off', ''), + reenqueueOn ? pill(crossNodeOn ? 'heal on · cross-node' : 'heal on', 'ok') : pill('heal off', ''), + ], + body: [ + el('div', { cls: 'stat-grid tight' }, [ + stat( + 'Refused', + num(refused), + touched + ? `${pct(refused, touched)} of touched serves · each an origin round trip` + : 'serves an invalidation cost', + { warn: refused > 0 && rescued === 0 && verificationOn } + ), + stat( + 'Rescued', + num(rescued), + touched + ? `${pct(rescued, touched)} · served on the probe’s evidence (verified)` + : 'served through an invalidation on evidence' + ), + stat( + 'Verifications recorded', + num(written), + writeFaults.length + ? `${writeFaults.map(([outcome, count]) => `${num(count)} ${outcome}`).join(' · ')}` + : 'by the probe sweep, while a scope is invalidated', + { warn: writeFaults.length > 0 } + ), + stat( + 'Heals attempted', + num(healTotal), + healTotal + ? `${num(heals.get(LOWERED) ?? 0)} lowered · ${pct(notOwner, healTotal)} not-owner` + : 'crawled invalidated pages pulled forward', + // The accelerator's real ceiling: traffic lands where the CDN sends it while residency + // is hashed, so on a concentrated deployment most heals arrive off-owner. Past half, + // cross-node forwarding is the lever. + { warn: healTotal > 0 && notOwner > healTotal / 2 && !crossNodeOn } + ), + ]), + healRows.length + ? barList( + healRows.map((row) => ({ + label: row.label, + value: row.count, + color: + row.tone === 'ok' + ? 'var(--ok)' + : row.tone === 'bad' + ? 'var(--bad)' + : row.tone === 'warn' + ? 'var(--warn)' + : row.tone === 'info' + ? 'var(--info)' + : SERIES[0], + sub: pct(row.count, healTotal), + title: `${row.label}: ${num(row.count)} — ${row.means}`, + })), + { format: fmtCount } + ) + : null, + errorTotal > 0 && + el('div', { cls: lkgExpired > 0 ? 'note bad' : 'note warn' }, [ + el('strong', { text: `${num(errorTotal)} epoch resolution failure(s) this hour: ` }), + [...errors.entries()].map(([kind, count]) => `${kind} ${num(count)}`).join(' · '), + lkgExpired > 0 + ? '. lkg-expired means the row read threw AND the last-known-good memory was older than invalidation.lkgMaxAge, so those requests failed OPEN — content someone deliberately invalidated may have been served.' + : '. read-error means a live last-known-good answered; invalid-row and unknown-mode were treated as hard invalidations.', + ]), + nothingActive && + !touched && + el('div', { cls: 'note' }, [ + 'Nothing is invalidated right now, so none of this is expected to move: a verification is only ', + 'written while a scope is invalidated, and a heal only fires on a serve an invalidation refused. ', + 'Zeros here are the steady state, not a fault.', + ]), + !nothingActive && + touched > 0 && + !verificationOn && + el('div', { cls: 'note' }, [ + 'Every touched serve was refused because ', + el('code', { text: 'invalidation.verification.enabled' }), + ' is off. With it on, a page the change probe has re-confirmed against the origin since the epoch is ', + 'served from cache instead of proxied — measured at 71-78% of a route-wide invalidation’s scope on ', + 'one deployment. It requires ', + el('code', { text: 'changeProbe.pageCheck' }), + ' on the rule whose scope was invalidated.', + ]), + el('p', { cls: 'muted chart-note' }, [ + 'Refused and rescued are the population an invalidation touched — pages rendered before the epoch that ', + 'a crawler asked for — split by what happened next. A refused serve is proxied to the origin (it is ', + 'also inside the miss/origin figures on Traffic); a rescued one is served from cache because the probe ', + 'proved its price and availability current, which is all “verified” asserts. The heal outcomes sum to ', + 'attempts; “forwarded” is a hand-off whose result the owner counts under its own verdict, so it is ', + 'never added to “lowered”. Counters, node-local, summed under cluster scope.', + ]), + ], + foot: [scanFooter(data)], + }); +} + function active(ctx, data) { const rows = (data.invalidations ?? []).map((row) => el('tr', null, [ @@ -211,7 +477,10 @@ const settings = (ctx) => 'How the rows in Active invalidations above are applied. enabled is the header pill: false leaves ' + 'every row stored and stops honouring all of them at once, so the corpus serves pre-invalidation ' + 'bytes again — it does not clear anything. maxScopes is the slot count on that panel. pad only ' + - 'widens the comparison toward invalidating (its cost is at most one extra render per page), and the ' + - 'reenqueue options decide whether a bot request for an invalidated page pulls that URL forward in ' + - 'the queue instead of waiting out its own cadence. Nothing here rewrites a cached page.', + 'widens the comparison toward invalidating (its cost is at most one extra render per page). ' + + 'verification lets a page the change probe has PROVED current since the epoch be served through ' + + 'the invalidation (the “rescued” count above; needs pageCheck on the rule). The reenqueue options ' + + 'decide whether a bot request for an invalidated page pulls that URL forward in the queue instead of ' + + 'waiting out its own cadence, and crossNode whether a heal for a key another node owns is forwarded ' + + 'to it rather than dropped as not-owner. Nothing here rewrites a cached page.', }); diff --git a/packages/console/src/admin/views/overview.js b/packages/console/src/admin/views/overview.js index 0389863e..83de21ad 100644 --- a/packages/console/src/admin/views/overview.js +++ b/packages/console/src/admin/views/overview.js @@ -42,8 +42,10 @@ import { colorFor, emptyNote, fmtMs, + fmtNet, isMerged, legend, + originLoad, pick, scopeLabel, stackBy, @@ -210,6 +212,9 @@ function traffic(ctx) { const total = sumCount(serves); const originServes = sumCount(serves.filter((s) => s.path === 'origin')); const cacheServes = sumCount(serves.filter((s) => s.path === 'cache')); + // The figure the gross tile flatters: renders, probes and sitemap fetches are origin requests + // too. Same arithmetic as Traffic (charts.js), so the two views cannot disagree about "net". + const load = originLoad(data); // The median, matching Traffic: an evenly refreshed corpus puts its p95 within a whisker of the // interval by construction, so a p95 tile here would read as "behind" on a healthy fleet. const ageMedian = weighted(pick(data, 'page_age'), 'median'); @@ -226,9 +231,17 @@ function traffic(ctx) { body: [ el('div', { cls: 'stat-grid tight' }, [ stat('Serves', num(total)), - stat('Origin offload', pct(total - originServes, total), null, { - warn: total > 0 && originServes > total / 2, - }), + stat( + 'Origin offload', + pct(total - originServes, total), + // Gross on the face, net underneath — the subtitle is what stops a 90% headline being + // quoted for a deployment whose renders and probes hand most of it back. + Number.isFinite(load.net) + ? `${fmtNet(load.net)} net of renders + probes · before crawler follow-up requests` + : 'gross — crawler requests not proxied live', + // Either figure under half is the flag; the net one is the one that can go negative. + { warn: (total > 0 && originServes > total / 2) || (Number.isFinite(load.net) && load.net < 0.5) } + ), stat('Cache-served', pct(cacheServes, total)), stat('Page age', fmtMs(ageMedian), 'median, cache serves only ≈', { warn: Number.isFinite(ageMedian) && Number.isFinite(interval) && ageMedian > interval, diff --git a/packages/console/src/admin/views/probe.js b/packages/console/src/admin/views/probe.js index 589dc1cf..bea46569 100644 --- a/packages/console/src/admin/views/probe.js +++ b/packages/console/src/admin/views/probe.js @@ -95,6 +95,29 @@ const sweepCadence = (sweep) => { return `continuous · ${target} · ${slice}`; }; +/** + * Nodes whose probe state row could not be read (plugin v0.62.0's `stateAvailable: false`), + * normalized over the node and cluster payload shapes. Empty for an older plugin, which has no + * such row and no such field. + */ +const stateUnavailableOn = (status) => + status.stateUnavailableOn ?? (status.stateAvailable === false ? [status.node ?? 'this node'] : []); + +/** + * "sweep running", with how far it has got when the plugin says. `examinedApprox` is the pass's + * heartbeat counter — rows walked so far, rounded to the yield interval — and it is the one number + * that separates a sweep that is moving from one that merely claims to be. Summed across the + * nodes running, because each walks its own disjoint slice. + */ +const sweepProgress = (status) => { + // One object at node scope, a per-node list under the cluster merge. + const progress = status.sweep?.progress; + const list = Array.isArray(progress) ? progress : progress ? [progress] : []; + const examined = list.reduce((acc, p) => acc + (Number.isFinite(p?.examinedApprox) ? p.examinedApprox : 0), 0); + return examined > 0 ? ` · ~${num(examined)} rows examined` : ''; +}; +const sweepRunningLabel = (status) => `sweep running${sweepProgress(status)}`; + /** * The pass counters, in the order a pass produces them. * @@ -236,6 +259,24 @@ function state(ctx, status) { ); } + // Plugin v0.62.0 publishes the probe's state to a node-local row so any worker can answer for + // the node; before it, 15 of 16 workers answered "not running, never ran" — indistinguishable + // from the probe being off. `stateAvailable: false` is the row failing to READ, and everything + // this page says about that node's sweep and canary is then untrustworthy rather than + // reassuring. Above the counters, because it changes what they mean. + const stateUnavailable = stateUnavailableOn(status); + if (stateUnavailable.length) { + body.push( + note('bad', [ + `The change-probe state row could not be read on ${stateUnavailable.join(', ')}. `, + 'Everything this page reports for ', + stateUnavailable.length > 1 ? 'those nodes' : 'that node', + ' — running or not, last pass, armed cadence — is unknown, not idle: the sweep may well be running. ', + 'The node’s log says why the read failed (a coordination database problem, usually).', + ]) + ); + } + body.push( kv([ ['Sweep', sweepCadence(status.sweep) ?? 'not armed'], @@ -289,7 +330,7 @@ function state(ctx, status) { : 'live', 'ok' ), - status.sweep?.running && pill('sweep running', 'info'), + status.sweep?.running && pill(sweepRunningLabel(status), 'info'), status.canary?.running && pill('canary running', 'info'), spacer(), segmented( @@ -697,7 +738,10 @@ function sweepCard(ctx, status) { head: [ sweepCadence(sweep) ? pill(sweepCadence(sweep), 'ok') : pill('not armed', 'bad'), sweep.running && - pill(sweep.runningOn?.length ? `running on ${sweep.runningOn.join(', ')}` : 'running now', 'info'), + pill( + `${sweep.runningOn?.length ? `running on ${sweep.runningOn.join(', ')}` : 'running now'}${sweepProgress(status)}`, + 'info' + ), spacer(), muted(last?.nodes > 1 ? `summed over ${last.nodes} nodes’ own slices` : ''), ], @@ -855,8 +899,17 @@ function nodeTable(status) { el('tr', null, [ el('td', { cls: 'mono', text: row.hostname }), el('td', null, [ + // An unreadable state row outranks everything else in the cell: "enabled" is read + // from config and is true, but running/last-pass/probed are all unknown for this node. + row.stateAvailable === false && pill('state unreadable', 'bad'), row.enabled === false ? pill('disabled', 'bad') : pill('enabled', 'ok'), - row.sweepRunning && pill('sweeping', 'info'), + row.sweepRunning && + pill( + Number.isFinite(row.sweepProgress?.examinedApprox) + ? `sweeping · ~${num(row.sweepProgress.examinedApprox)} examined` + : 'sweeping', + 'info' + ), row.canaryRunning && pill('canary', 'info'), ]), el('td', null, [row.dryRun === false ? pill('live', 'ok') : pill('dry run', 'warn')]), diff --git a/packages/console/src/admin/views/traffic.js b/packages/console/src/admin/views/traffic.js index 9f42d82f..2c42fbb8 100644 --- a/packages/console/src/admin/views/traffic.js +++ b/packages/console/src/admin/views/traffic.js @@ -18,6 +18,19 @@ * it costs no request), falling back to `render.defaultInterval`. Absolute milliseconds stay one * click away, and the KPI carries both. * + * OFFLOAD IS STATED TWICE, GROSS AND NET, because the number the rollout is judged on is not the + * one the serve path can see. Gross offload is the share of crawler requests that were not proxied + * to the origin live — `bot_serve` source ≠ origin — and it is what every serve-side panel here is + * built from. It is also flattering: it counts what the origin was spared and none of what this + * system asks of the origin in exchange — every render, every change probe, every sitemap fetch is + * an origin request. Net offload subtracts those (`originLoad` in charts.js carries the arithmetic + * and its caveats), and the panel beside the origin-fetch chart shows each term. One term it + * cannot count is stated rather than omitted: the requests a page's own scripts make when a + * rendering crawler runs it, which never pass through this plugin and are missing from BOTH sides + * — the origin would have served them for every raw page-view, and does not for a snapshot served + * without scripts. The net tile says "before crawler follow-up requests" and the panel reports the + * exposure — every page handed to a crawler — as a count, never multiplied by a guessed factor. + * * AND NOT EVERY MISS IS OURS. A miss whose origin fetch came back 404 or 410 is a URL that does * not exist anywhere — there is nothing for the cache to be missing, and it can never improve, * because only a 200 is ever scheduled for prerendering. Left in the coverage number those URLs @@ -39,9 +52,11 @@ * THE BOT FILTER IS CLIENT-SIDE, ALWAYS. Selecting bots re-renders from the payload already in * hand; it never refetches, because the load discipline below is the whole reason this view can * afford to be this detailed. Four metrics carry a bot (`bot_request`, `bot_serve`, `page_age`, - * and `prerender_ops`'s `discovery_gated`) and the rest do not — `route_serve`, `origin_fetch`, `duration` and `response_*` have no bot - * dimension at all — so a panel that CANNOT honour the filter says "all bots" on its face rather - * than quietly showing every crawler's numbers under one crawler's name. + * and `prerender_ops`'s `discovery_gated`) and the rest do not — `route_serve`, `origin_fetch`, + * `render`, the probe and sitemap counters, `duration` and `response_*` have no bot dimension at + * all — so a panel that CANNOT honour the filter says "all bots" on its face rather than quietly + * showing every crawler's numbers under one crawler's name. Net offload is one of those: a render + * is not for any one crawler. * * LOAD DISCIPLINE. Every number on this view comes from a single `analytics` request; each * node answers it from a per-worker cache inside `management.analytics.cacheTtl`, so switching @@ -69,9 +84,13 @@ import { emptyNote, fmtCount, fmtMs, + fmtNet, fmtRatio, + isCacheServed, legend, lineChart, + originLoad, + originLoadBuckets, pick, rangePicker, ratioOf, @@ -148,7 +167,9 @@ export function render(ctx) { const requests = pick(data, 'bot_request', (s) => keepBot(filter, s.method)); const ages = pick(data, 'page_age', (s) => keepBot(filter, s.path)); const cadences = cadenceIndex(configState(ctx).payload, data.intervals?.defaultRenderInterval); - const scope = { serves, requests, ages, cadences, filter }; + // Over ALL bots whatever the filter says: three of its four terms carry no bot (see the header). + const load = originLoad(data); + const scope = { serves, requests, ages, cadences, filter, load }; return [ head, @@ -158,8 +179,10 @@ export function render(ctx) { kpis(data, scope), el('div', { cls: 'cols' }, [freshness(data, scope), staleness(ctx, data, scope)]), notFreshHit(data, scope), - el('div', { cls: 'cols' }, [originFetch(data, filter), latency(data, filter)]), - el('div', { cls: 'cols' }, [crawlers(data, scope), statusCodes(data, filter)]), + // The two origin-side panels together: what the origin was asked, then what each ask cost. + el('div', { cls: 'cols' }, [originSeen(data, scope), originFetch(data, filter)]), + el('div', { cls: 'cols' }, [latency(data, filter), statusCodes(data, filter)]), + crawlers(data, scope), routes(ctx, data, cadences, filter), discoveryGate(ctx, data, filter), breadth(ctx, filter), @@ -377,6 +400,7 @@ function kpis(data, scope) { const freshHits = sumCount(serves.filter((s) => s.method === 'hit')); const coverage = coverageSplit({ serves, costs: originCostByReason(data), filter }); const arrived = sumCount(requests); + const { load } = scope; // SERVE TIME IS TWO POPULATIONS, and one number over both is a number about the hit rate. A // cache hit is single-digit milliseconds and an origin proxy is hundreds; pooling them produces @@ -420,12 +444,27 @@ function kpis(data, scope) { { warn: unresolvedShare > 0.02 } ), stat( - 'Origin offload', + 'Origin offload · gross', pct(total - originServes, total), - 'requests the origin never saw', + 'crawler requests not proxied live', // The offload number is the rollout's headline; a majority-origin window deserves the flag. { warn: total > 0 && originServes > total / 2 } ), + stat( + 'Origin offload · net', + fmtNet(load.net), + // The subtitle carries the sum, because the tile's whole point is what the gross figure + // leaves out — and under a bot filter it says so, since this one cannot narrow. "Before + // crawler follow-up requests" is the term that is NOT in the sum (see originLoad). + load.arrived > 0 + ? `origin saw ${fmtCount(load.total)} of ${fmtCount(load.arrived)} asked · before crawler follow-up requests${ + filter ? ' · all bots' : '' + }` + : 'no crawler requests in the window', + // Below half is a flag on either figure; below ZERO means this system is sending the origin + // more requests than the crawlers would have on their own — the finding, not a display bug. + { warn: Number.isFinite(load.net) && load.net < 0.5 } + ), stat('Cache-served', pct(cacheServes, total), 'stored snapshot answered'), stat('Fresh hits', pct(freshHits, total), 'inside the configured cadence'), stat( @@ -473,9 +512,9 @@ function freshness(data, { serves, filter }) { ? stackedBars(data, keys, stacks, (k) => colorFor(CACHE_STATUS_COLORS, k)) : emptyNote('bot_serve', data), el('p', { cls: 'muted chart-note' }, [ - 'hit + swr + peer-rescue is cache-served. A rising miss share is a coverage problem; a rising ', - 'swr share is the fleet not keeping the configured cadence; blob-* should sit at zero. What each ', - 'verdict costs, and which of them are the same problem, is the panel below.', + 'hit + swr + verified + peer-rescue is cache-served. A rising miss share is a coverage problem; a ', + 'rising swr share is the fleet not keeping the configured cadence; blob-* should sit at zero. What ', + 'each verdict costs, and which of them are the same problem, is the panel below.', ]), filter && muted(`Filtered to ${[...filter].join(', ')}.`), ], @@ -534,9 +573,7 @@ function staleness(ctx, data, scope) { // differ whenever a target's cadence comes from its stored value (a sitemap `changefreq`) // instead — a case no metric exposes. When the two disagree, the verdicts win and the divisor // is what is wrong, so say that rather than let a config gap read as a fleet failure. - const cacheServed = sumCount( - serves.filter((x) => x.method === 'hit' || x.method === 'swr' || x.method === 'peer-rescue') - ); + const cacheServed = sumCount(serves.filter((x) => isCacheServed(x.method))); const pastDue = sumCount(serves.filter((x) => x.method === 'swr' || x.method === 'stale')); const contradicted = normalizable && Number.isFinite(ratioP95) && ratioP95 > 1 && cacheServed > 0 && pastDue / cacheServed < 0.01; @@ -643,8 +680,12 @@ const FAMILIES = [ }, { key: 'invalidated', - label: 'Invalidated', - hint: 'a bulk invalidation cost the serve', + label: 'Invalidation', + // One family for both halves, because they are one population — the pages a bulk + // invalidation touched — split by what happened next: refused (`invalidated`, proxied to the + // origin) or rescued (`verified`, served from cache on the probe's evidence). Read against + // each other they size what per-page verification is buying; apart, neither means much. + hint: 'a bulk invalidation touched the serve — refused, or rescued on evidence', }, { key: 'not-cacheable', @@ -669,6 +710,13 @@ const NOT_HIT = { 'the local body failed and the residency owner’s copy answered it — still a cache serve', ], 'invalidated': ['invalidated', 'a bulk invalidation demoted a page that would otherwise have served'], + // Deliberately its own verdict upstream and never folded into `hit`: the page is as old as it + // ever was and is being served on EVIDENCE (the probe re-confirmed its price/availability + // claims after the epoch), not on age. Still a cache serve — it counts toward offload. + 'verified': [ + 'invalidated', + 'an invalidation would have refused it; the change probe proved its claims current — still a cache serve', + ], 'skip': ['not-cacheable', 'the cache was deliberately not consulted (renderNow / Cache-Control)'], 'bypass': ['not-cacheable', 'not a cacheable request at all (non-GET/HEAD)'], }; @@ -902,9 +950,10 @@ function notFreshHit(data, { serves, filter }) { 'counted in the status panel below.', ]), el('p', { cls: 'muted chart-note' }, [ - 'One row per freshness verdict, because they are four different problems: coverage is fixed in the ', + 'One row per freshness verdict, because they are different problems: coverage is fixed in the ', 'corpus (discovery, sitemaps), cadence by render capacity or a longer interval, integrity is blob ', - 'health and never a caching question, and the last two are working as configured. ', + 'health and never a caching question, invalidation is a bulk invalidation doing its job — with ', + '`verified` the part per-page verification bought back — and not-cacheable is working as configured. ', '“origin median” is what that verdict typically costs at the origin, with its p95 in the tooltip — and ', '“origin answered” is what came back: `absent` (404/410) is a page nobody has, `server-error` and ', '`connect-fail` are the origin in trouble, and only `served` is a page we could have had cached.', @@ -1059,6 +1108,143 @@ function statusCodes(data, filter) { }); } +// ---- what the origin actually saw ---------------------------------------------- +// +// The gross offload tile is a serve-side number, and the serve path is not the only thing here +// that talks to the origin. The arithmetic is `originLoad` in charts.js (the overview reads the +// same figure); this is the panel that shows its terms. + +/** What the origin saw, by cause, over time — the panel behind the net offload tile. */ +const LOAD_ROWS = [ + { + key: 'proxied', + label: 'proxied serves', + color: 'var(--warn)', + means: 'crawler requests forwarded live — the only term the gross figure counts', + }, + { + key: 'renders', + label: 'renders', + color: SERIES[0], + means: 'page loads by the render fleet, one per posted result — the document only', + }, + { + key: 'probes', + label: 'change probes', + color: SERIES[2], + means: 'origin calls by the change probe, failures included', + }, + { key: 'sitemaps', label: 'sitemap fetches', color: SERIES[3], means: 'sitemaps read by refresh runs' }, +]; + +function originSeen(data, { load, filter }) { + const stacks = originLoadBuckets(data); + const present = LOAD_ROWS.filter((row) => load[row.key] > 0); + const keys = present.map((row) => row.key); + const colorOf = (key) => LOAD_ROWS.find((row) => row.key === key)?.color ?? 'var(--fg-4)'; + + return card('What the origin actually saw', { + head: [ + spacer(), + allBotsTag(filter, 'renders, probes and sitemap fetches are not for any one crawler'), + legend(present.map((row) => ({ label: row.label, color: row.color }))), + ], + body: !load.total + ? [ + el('div', { cls: 'note ok' }, [ + 'Nothing in this window reached the origin — no proxied serve, no render, no probe, no sitemap ', + 'fetch. Offload is 100% gross and net alike.', + ]), + followUpNote(load), + ] + : [ + el('div', { cls: 'stat-grid tight' }, [ + stat('Crawlers asked for', fmtCount(load.arrived), 'requests at ingress, all bots'), + stat( + 'The origin answered', + fmtCount(load.total), + `${pct(load.total, load.arrived)} of that — ${fmtNet(load.net)} net offload`, + { warn: Number.isFinite(load.net) && load.net < 0.5 } + ), + stat( + 'Gross offload counts', + fmtCount(load.proxied), + `${fmtCount(load.total - load.proxied)} more came from this system` + ), + // The unmeasured term gets a tile so it sits in the same row as the measured ones, at + // the same size — not a footnote under a number that looks complete without it. + stat( + 'Crawler follow-up requests', + 'not measured', + `${fmtCount(load.handed)} pages handed to crawlers · counted on neither side of the ledger` + ), + ]), + stackedBars(data, keys, stacks, colorOf, { format: fmtCount }), + barList( + present.map((row) => ({ + label: row.label, + value: load[row.key], + color: row.color, + sub: pct(load[row.key], load.total), + title: `${row.label}: ${num(load[row.key])} — ${row.means}`, + })), + { format: fmtCount } + ), + el('p', { cls: 'muted chart-note' }, [ + 'Every request the origin answered because this deployment exists, against what crawlers asked ', + 'for. Gross offload counts only the first bar; net offload subtracts all of them. A render is ', + 'counted as ONE request — the document; the page’s own scripts and stylesheets reach the origin ', + 'too if the CDN does not cache them for the renderer, which nothing here can see. A probe is one ', + 'small endpoint call, not a page render — cheaper per request than the others, but a request.', + load.lumpy + ? ' Probe and sitemap counts land where a PASS FINISHED, not where the requests happened: over ' + + 'a range shorter than a sweep this is either none of a running pass or all of one that just ' + + 'ended. Quote the 24h figure.' + : '', + ' Not counted: requests the CDN answered from its own cache, and renders that never posted a result.', + ]), + followUpNote(load), + Number.isFinite(load.net) && + load.net < 0 && + el('div', { cls: 'note warn' }, [ + el('strong', { text: 'The origin saw more requests than the crawlers made. ' }), + 'Net offload is negative: the render and probe cadence is generating more origin load than ', + 'the bot traffic it is standing in for. That is the expected shape of a fresh deployment ', + 'backfilling its corpus, and of a corpus much larger than what crawlers actually walk — the ', + 'levers are the render interval, the demand ladder’s floors, the discovery gate, and the ', + 'probe rate. Read it over 24h before acting on it.', + ]), + ], + }); +} + +/** + * The fifth term, spelled out. Shown whenever a page was handed to a crawler at all — including on + * the "nothing reached the origin" exit, where a net offload of 100% is precisely the claim this + * term qualifies (in that case, upward). + */ +function followUpNote(load) { + if (!load.handed) return null; + return el('div', { cls: 'note' }, [ + el('strong', { + text: `${num(load.handed)} pages were handed to crawlers, and the requests their scripts make are counted on neither side. `, + }), + 'A rendering crawler (Googlebot, Bingbot, Applebot) fetches a page and then runs it, and the page makes ', + 'its own XHR/fetch calls to the origin — inventory, pricing, personalisation — which are exactly the ', + 'calls no CDN caches. Without this system every such crawl costs the origin the document PLUS those ', + 'calls, so the “crawlers asked for” baseline above understates what the origin was spared. With it, a ', + 'snapshot served without its scripts triggers none of them (a saving not credited above), a snapshot ', + 'that keeps its scripts or a proxied origin page triggers them as before (a cost not charged), and our ', + 'own renders run the page too (each render is really the document plus those calls). None of it passes ', + 'through this plugin — the CDN sends a crawler’s subrequests straight to the origin — so the figure is ', + 'documents-only on both sides and says so, for every crawler rather than a guessed subset. Where ', + 'snapshots are served with scripts stripped, the true net offload for rendering crawlers is HIGHER than ', + 'shown. The render fleet can measure the per-page factor (it loads the same pages and already classifies ', + 'every same-origin response as cacheable or not); applied to both sides by what the registry says each ', + 'crawler runs, this becomes a counted term.', + ]); +} + /** What a non-cache serve costs: why the origin was consulted, how slowly it answered, and with what. */ function originFetch(data, filter) { const fetches = pick(data, 'origin_fetch'); @@ -1114,7 +1300,7 @@ function routes(ctx, data, cadences, filter) { const route = s.path ?? 'unrouted'; const entry = byRoute.get(route) ?? { total: 0, cache: 0, miss: 0, aging: 0, integrity: 0 }; entry.total += s.count; - if (s.method === 'hit' || s.method === 'swr' || s.method === 'peer-rescue') entry.cache += s.count; + if (isCacheServed(s.method)) entry.cache += s.count; if (s.method === 'miss') entry.miss += s.count; if (s.method === 'swr' || s.method === 'stale') entry.aging += s.count; if (s.method === 'blob-missing' || s.method === 'blob-timeout' || s.method === 'peer-rescue') diff --git a/packages/console/src/util/aggregate.js b/packages/console/src/util/aggregate.js index 3f83accb..69102a1b 100644 --- a/packages/console/src/util/aggregate.js +++ b/packages/console/src/util/aggregate.js @@ -1098,9 +1098,23 @@ export function mergeChangeProbe(results) { ownerScopeNote: bodies[0].b.ownerScopeNote ?? null, rules, rulesDiverge: new Set(ruleJson).size > 1, + // Plugin v0.62.0 publishes the probe's state to a node-local row every worker can read, + // and says when that row could NOT be read: `stateAvailable: false` is "this answer is not + // trustworthy", which the old per-worker shape could not express (it reported `running: + // false, lastRun: null` — indistinguishable from the probe being off). Named per node, + // because a node whose row is unreadable contributes zeros to every sum above exactly the + // way an unswept node does, and the UI has to say which it is. An older plugin that has no + // such field is not flagged: absent is "not reported", not "unavailable". + stateUnavailableOn: bodies.filter((r) => r.b.stateAvailable === false).map((r) => r.hostname), sweep: { running: bodies.some((r) => r.b.sweep?.running), runningOn: bodies.filter((r) => r.b.sweep?.running).map((r) => r.hostname), + // Live progress of the passes in flight, per node — a sweep runs for hours, and "running" + // alone cannot tell a pass that is moving from one whose worker died (the plugin's own + // heartbeat staleness handles the latter; this is the number an operator watches move). + progress: bodies + .filter((r) => r.b.sweep?.running && r.b.sweep?.progress) + .map((r) => ({ hostname: r.hostname, ...r.b.sweep.progress })), armedInterval: bodies.find((r) => Number.isFinite(r.b.sweep?.armedInterval))?.b.sweep.armedInterval ?? null, sweptNodes: sweeps.length, unsweptNodes: bodies.filter((r) => !r.b.sweep?.lastRun?.startedAt).map((r) => r.hostname), @@ -1157,7 +1171,10 @@ export function mergeChangeProbe(results) { hostname: r.hostname, enabled: r.b.enabled ?? null, dryRun: r.b.dryRun ?? null, + // null on a plugin that predates the shared state row; false is the finding. + stateAvailable: typeof r.b.stateAvailable === 'boolean' ? r.b.stateAvailable : null, sweepRunning: !!r.b.sweep?.running, + sweepProgress: r.b.sweep?.running ? (r.b.sweep?.progress ?? null) : null, canaryRunning: !!r.b.canary?.running, sweepFinishedAt: msOf(r.b.sweep?.lastRun?.finishedAt) || null, canaryFinishedAt: msOf(r.b.canary?.lastRun?.finishedAt) || null, diff --git a/packages/console/test/adminAssets.test.js b/packages/console/test/adminAssets.test.js index 9c86b317..0ca2a3fd 100644 --- a/packages/console/test/adminAssets.test.js +++ b/packages/console/test/adminAssets.test.js @@ -257,12 +257,13 @@ test('a metric the plugin emits is charted by the console, or waived with a reas // A series may legitimately have no panel — but it has to be a decision, written down here, // not an oversight nobody noticed. + // `invalidation_error` and `invalidation_reenqueue` were waived here until console v0.12.0 — the + // Invalidations view now reads both, beside `page_verification` and the `verified` serve status + // they belong with. This test is what said the console was blind to `page_verification`. const NOT_CHARTED = new Map([ ['prerender_ops.serve_error', 'blob-fault counter; the serve-side view of it is bot_serve blob-* on Traffic'], ['prerender_ops.unrouted', 'has its own endpoint and panel (the unrouted report), not the analytics window'], ['prerender_ops.config_warnings', 'the Config view reads the warnings themselves, which say more than a count'], - ['prerender_ops.invalidation_error', 'the Invalidations view reads the rows; a failure count adds nothing'], - ['prerender_ops.invalidation_reenqueue', 'demand-driven heal is off by default and has no panel yet'], ]); const client = [...clientSources.values()].join('\n'); diff --git a/packages/console/test/aggregate.test.js b/packages/console/test/aggregate.test.js index 90c21a58..e39df3fe 100644 --- a/packages/console/test/aggregate.test.js +++ b/packages/console/test/aggregate.test.js @@ -1107,3 +1107,47 @@ test('discovery purge: a self-stopped pass is reported with the samples that exp assert.equal(body.errorSamples[0].hostname, 'b.example.com:9926'); assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').abortedOnErrors, true); }); + +// Plugin v0.62.0 moved the probe's state into a node-local row every worker can read, and made +// the endpoint say when that row could NOT be read. The merge has to carry that through: a node +// whose state is unreadable contributes zeros to every sum exactly the way an unswept node does, +// and only a name separates "idle" from "unknown". +test('change probe: a node whose state row could not be read is NAMED, not read as idle', () => { + const { body } = mergerFor('change-probe')([ + ok('a', probeBody('a', { stateAvailable: true })), + ok('b', probeBody('b', { stateAvailable: false, sweep: { running: false, armedInterval: null, lastRun: null } })), + ]); + assert.deepEqual(body.stateUnavailableOn, ['b.example.com:9926']); + assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').stateAvailable, false); + assert.equal(body.byNode.find((row) => row.hostname === 'a.example.com:9926').stateAvailable, true); +}); + +test('change probe: a plugin that predates the state row is not flagged as unavailable', () => { + const { body } = mergerFor('change-probe')([ok('a', probeBody('a')), ok('b', probeBody('b'))]); + assert.deepEqual(body.stateUnavailableOn, []); + assert.equal(body.byNode[0].stateAvailable, null); +}); + +test('change probe: live progress rides along per node, and only while the pass is running', () => { + const { body } = mergerFor('change-probe')([ + ok( + 'a', + probeBody('a', { + sweep: { running: true, armedInterval: 86_400_000, lastRun: null, progress: { examinedApprox: 12_000 } }, + }) + ), + // A finished pass that still carries a stale progress object must not be reported as moving. + ok( + 'b', + probeBody('b', { + sweep: { running: false, armedInterval: 86_400_000, lastRun: null, progress: { examinedApprox: 5 } }, + }) + ), + ]); + assert.deepEqual(body.sweep.progress, [{ hostname: 'a.example.com:9926', examinedApprox: 12_000 }]); + assert.deepEqual(body.sweep.runningOn, ['a.example.com:9926']); + assert.deepEqual(body.byNode.find((row) => row.hostname === 'a.example.com:9926').sweepProgress, { + examinedApprox: 12_000, + }); + assert.equal(body.byNode.find((row) => row.hostname === 'b.example.com:9926').sweepProgress, null); +}); diff --git a/packages/console/test/invalidationsView.test.js b/packages/console/test/invalidationsView.test.js new file mode 100644 index 00000000..fb5d140a --- /dev/null +++ b/packages/console/test/invalidationsView.test.js @@ -0,0 +1,289 @@ +/** + * The Invalidations view, EXECUTED — its "what the invalidations are doing" panel above all. + * + * That panel reads three analytics series nobody read before console v0.12.0 (`page_verification`, + * `invalidation_reenqueue`, `invalidation_error`) beside the `verified` serve status, and the + * arithmetic has two rules a reader cannot verify from the screen: + * + * - `forwarded` is NEVER added to `lowered`. A forwarded heal is counted again by the owner under + * its own verdict in the same series, so under cluster scope adding the two double-counts. + * - Refused and rescued are ONE population split two ways, and the shares are stated against + * that population, not against all serves. + * + * The rest is the empty states: nothing invalidated is a steady state, not a fault; verification + * off explains a 100% refusal rate rather than letting it read as the feature failing. + */ + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { installDom, find } from './domShim.js'; + +installDom(); + +const { el } = await import('../src/admin/ui.js'); +const { load, render } = await import('../src/admin/views/invalidations.js'); + +const HOUR = 3_600_000; + +/** One analytics combo, in the shape `util/analyticsRead.js` emits (four buckets, flat). */ +const combo = (metric, path, method, type, count) => ({ + metric, + path, + method, + type, + count, + total: 0, + counts: [count / 4, count / 4, count / 4, count / 4], +}); + +const LIST = { + enabled: true, + maxScopes: 8, + knownScopes: ['all', 'route:prefix:/product/'], + invalidations: [ + { + scope: 'route:prefix:/product/', + resolvable: true, + invalidatedAt: new Date(Date.now() - HOUR).toISOString(), + reason: 'promo flip', + updatedBy: 'ops', + }, + ], +}; + +const EMPTY_LIST = { ...LIST, invalidations: [] }; + +const ANALYTICS = { + available: true, + scope: 'node', + node: 'node-a', + rangeMs: HOUR, + startMs: 0, + endMs: HOUR, + bucketMs: HOUR / 4, + bucketCount: 4, + scan: { ms: 4, scanned: 100, kept: 20, cap: 20_000 }, + series: [ + // 300 refused, 100 rescued: 75% / 25% of the population the invalidation touched. + combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300), + combo('bot_serve', 'cache', 'verified', 'googlebot', 100), + combo('bot_serve', 'cache', 'hit', 'googlebot', 5000), + // Verifications the sweep recorded, one of them failing to write. + combo('prerender_ops', 'page_verification', 'written', null, 4000), + combo('prerender_ops', 'page_verification', 'write-error', null, 3), + // Heal outcomes. 60 lowered on THIS node, 40 forwarded to owners (who count their own), the + // classic not-owner ceiling on the rest. + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'route:prefix:/product/', 60), + combo('prerender_ops', 'invalidation_reenqueue', 'forwarded', 'route:prefix:/product/', 40), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'route:prefix:/product/', 100), + combo('prerender_ops', 'invalidation_reenqueue', 'not-sooner', 'route:prefix:/product/', 20), + combo('prerender_ops', 'invalidation_reenqueue', 'forward-failed', 'route:prefix:/product/', 5), + ], +}; + +const config = (over = {}) => ({ + schema: { + children: { + invalidation: { + children: { + enabled: { kind: 'option' }, + verification: { children: { enabled: { kind: 'option' } } }, + reenqueue: { + children: { enabled: { kind: 'option' }, crossNode: { children: { enabled: { kind: 'option' } } } }, + }, + }, + }, + }, + }, + layers: [ + { path: 'invalidation.enabled', effective: true }, + { path: 'invalidation.verification.enabled', effective: over.verification ?? true }, + { path: 'invalidation.reenqueue.enabled', effective: over.reenqueue ?? true }, + { path: 'invalidation.reenqueue.crossNode.enabled', effective: over.crossNode ?? false }, + ], +}); + +function makeCtx({ list = LIST, analytics = ANALYTICS, cfg = config() } = {}) { + const views = {}; + const scratch = (id) => (views[id] ??= {}); + const calls = { gets: [], posts: [] }; + return { + calls, + scratch, + busy: false, + get data() { + return scratch('invalidations'); + }, + async get(route) { + calls.gets.push(route); + if (route === 'invalidations') return { ok: true, body: list }; + if (route === 'analytics') + return analytics ? { ok: true, body: analytics } : { ok: false, status: 500, body: {} }; + if (route === 'config') return { ok: true, body: cfg }; + return { ok: true, body: null }; + }, + async post(route, data) { + calls.posts.push({ route, data }); + return { ok: true, body: {} }; + }, + async run(fn) { + return fn(); + }, + fail() {}, + render() {}, + reload() {}, + go() {}, + }; +} + +const draw = (ctx) => el('div', null, render(ctx)); +const tile = (ctx, label) => + find(draw(ctx), (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === label); + +const ready = async (options) => { + const ctx = makeCtx(options); + await load(ctx); + return ctx; +}; + +test('the view reads the shared one-hour analytics window alongside the rows', async () => { + const ctx = await ready(); + assert.ok(ctx.calls.gets.includes('analytics')); + assert.match(draw(ctx).textContent, /What the invalidations are doing/); +}); + +test('refused and rescued are one population, and the shares are stated against it', async () => { + const ctx = await ready(); + const refused = tile(ctx, 'Refused'); + const rescued = tile(ctx, 'Rescued'); + assert.ok(refused && rescued, 'expected both tiles'); + assert.match(refused.textContent, /300/); + assert.match(refused.textContent, /75% of touched serves/); + assert.match(rescued.textContent, /100/); + assert.match(rescued.textContent, /25%/); + // 5,000 ordinary hits are NOT in the denominator: 300 of 5,400 would read "6%". + assert.doesNotMatch(refused.textContent, /\b6%/); +}); + +test('verification writes are counted, and a write fault is on the tile rather than in a log', async () => { + const ctx = await ready(); + const writes = tile(ctx, 'Verifications recorded'); + assert.match(writes.textContent, /4,000/); + assert.match(writes.textContent, /3 write-error/); + assert.ok( + find(writes, (n) => n.attributes?.class === 'value warn'), + 'a write fault should warn' + ); +}); + +test('forwarded heals are never added to lowered — the owner counts them under its own verdict', async () => { + const ctx = await ready(); + const heals = tile(ctx, 'Heals attempted'); + // 60 + 40 + 100 + 20 + 5 attempts; 60 lowered, not 100. + assert.match(heals.textContent, /225/); + assert.match(heals.textContent, /60 lowered/); + assert.match(heals.textContent, /44% not-owner/); + const text = draw(ctx).textContent; + for (const outcome of ['lowered', 'forwarded', 'not-owner', 'not-sooner', 'forward-failed']) { + assert.match(text, new RegExp(outcome)); + } +}); + +test('a not-owner majority warns only while cross-node forwarding is off — it is the lever', async () => { + const off = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'all', 10), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'all', 90), + ], + }, + }); + assert.ok( + find(tile(off, 'Heals attempted'), (n) => n.attributes?.class === 'value warn'), + 'should warn with crossNode off' + ); + + const on = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_reenqueue', 'lowered', 'all', 10), + combo('prerender_ops', 'invalidation_reenqueue', 'not-owner', 'all', 90), + ], + }, + cfg: config({ crossNode: true }), + }); + assert.equal( + find(tile(on, 'Heals attempted'), (n) => n.attributes?.class === 'value warn'), + null + ); + assert.match(draw(on).textContent, /heal on · cross-node/); +}); + +test('an outcome this console has never heard of still gets a row — the list must sum to the attempts', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('prerender_ops', 'invalidation_reenqueue', 'teleported', 'all', 7)] }, + }); + const text = draw(ctx).textContent; + assert.match(text, /teleported/); + assert.match(tile(ctx, 'Heals attempted').textContent, /7/); +}); + +test('lkg-expired is the bad note: those requests failed OPEN', async () => { + const ctx = await ready({ + analytics: { + ...ANALYTICS, + series: [ + combo('prerender_ops', 'invalidation_error', 'read-error', null, 4), + combo('prerender_ops', 'invalidation_error', 'lkg-expired', null, 2), + ], + }, + }); + const node = draw(ctx); + const bad = find(node, (n) => n.attributes?.class === 'note bad' && n.textContent.includes('epoch resolution')); + assert.ok(bad, 'expected a bad note'); + assert.match(bad.textContent, /6 epoch resolution failure/); + assert.match(bad.textContent, /failed OPEN/); +}); + +test('a read-error alone is a warning, not the open-failure alarm', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('prerender_ops', 'invalidation_error', 'read-error', null, 4)] }, + }); + const node = draw(ctx); + assert.ok(find(node, (n) => n.attributes?.class === 'note warn' && n.textContent.includes('epoch resolution'))); + assert.doesNotMatch(node.textContent, /failed OPEN/); +}); + +test('nothing invalidated and nothing touched is the steady state, and the panel says so', async () => { + const ctx = await ready({ + list: EMPTY_LIST, + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'cache', 'hit', 'googlebot', 500)] }, + }); + assert.match(draw(ctx).textContent, /Zeros here are the steady state, not a fault/); +}); + +test('every serve refused with verification OFF is explained as the setting, not the feature failing', async () => { + const ctx = await ready({ + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300)] }, + cfg: config({ verification: false }), + }); + const text = draw(ctx).textContent; + assert.match(text, /verification off/); + assert.match(text, /Every touched serve was refused because/); + // With verification ON and nothing rescued, the refused tile warns instead. + const on = await ready({ + analytics: { ...ANALYTICS, series: [combo('bot_serve', 'origin', 'invalidated', 'googlebot', 300)] }, + }); + assert.ok(find(tile(on, 'Refused'), (n) => n.attributes?.class === 'value warn')); +}); + +test('an analytics window that failed to load leaves the rest of the view standing', async () => { + const ctx = await ready({ analytics: null }); + const text = draw(ctx).textContent; + assert.match(text, /Active invalidations/); + assert.match(text, /Record an invalidation/); + assert.match(text, /No bot_serve rows/); +}); diff --git a/packages/console/test/overviewView.test.js b/packages/console/test/overviewView.test.js index d88fcda9..e4467d77 100644 --- a/packages/console/test/overviewView.test.js +++ b/packages/console/test/overviewView.test.js @@ -58,7 +58,7 @@ const CENSUS = { canceled: false, }; -function makeCtx({ overview = OVERVIEW, purge = { running: false } } = {}) { +function makeCtx({ overview = OVERVIEW, purge = { running: false }, analytics = ANALYTICS } = {}) { const views = {}; const scratch = (id) => (views[id] ??= {}); const calls = { posts: [] }; @@ -71,7 +71,7 @@ function makeCtx({ overview = OVERVIEW, purge = { running: false } } = {}) { }, async get(route) { if (route === 'overview') return { ok: true, body: overview }; - if (route === 'analytics') return { ok: true, body: ANALYTICS }; + if (route === 'analytics') return { ok: true, body: analytics }; if (route === 'discovery-purge') return { ok: true, body: purge }; return { ok: true, body: null }; }, @@ -323,3 +323,51 @@ test('a cluster where only some nodes spared bot-visited targets is called out', assert.match(text, /Only a\.example\.com:9926 spared bot-visited targets/); assert.match(text, /sum two different predicates/); }); + +// ---- the serve strip's offload tile ----------------------------------------------- + +/** One analytics combo, in the shape `util/analyticsRead.js` emits (four buckets, flat). */ +const combo = (metric, path, method, type, count, value) => ({ + metric, + path, + method, + type, + count, + total: 0, + counts: [count / 4, count / 4, count / 4, count / 4], + ...(value === undefined ? {} : { mean: value, median: value, p95: value, means: [value, value, value, value] }), +}); + +test('the offload tile shows the gross figure with the net one underneath, from the same arithmetic as Traffic', async () => { + // 900 of 1,000 serves from cache (90% gross); 500 renders and a 100-probe pass mean the origin + // answered 700 of 1,000 crawler requests — 30% net. The headline alone would be the flattering + // number; the subtitle is what stops it being quoted on its own. + const analytics = { + ...ANALYTICS, + startMs: 0, + endMs: 3_600_000, + bucketMs: 900_000, + bucketCount: 4, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 900), + combo('bot_serve', 'origin', 'miss', 'googlebot', 100), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 1000), + combo('render', 'outcome', 'rendered', null, 500), + combo('prerender_ops', 'probe_probed', null, null, 1, 100), + ], + }; + const ctx = await ready({ analytics }); + const tile = find( + draw(ctx), + (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === 'Origin offload' + ); + assert.ok(tile, 'expected an Origin offload tile'); + assert.match(tile.textContent, /90%/); + assert.match(tile.textContent, /30% net of renders \+ probes/); + assert.match(tile.textContent, /before crawler follow-up requests/); + // Below half on the net figure warns even though the gross one is fine. + assert.ok( + find(tile, (n) => n.attributes?.class === 'value warn'), + 'a 30% net offload should warn' + ); +}); diff --git a/packages/console/test/probeView.test.js b/packages/console/test/probeView.test.js index ea6791bb..c6da79a8 100644 --- a/packages/console/test/probeView.test.js +++ b/packages/console/test/probeView.test.js @@ -723,3 +723,102 @@ test('the local governor is reported apart from the origin one — the fixes poi assert.match(text, /local load/); assert.match(text, /event loop 180ms behind/); }); + +// ---- the shared state row (plugin v0.62.0) ------------------------------------- + +test('an unreadable state row is the loudest thing on the page — the node is unknown, not idle', async () => { + const ctx = await ready({ status: { ...STATUS, stateAvailable: false } }); + const text = draw(ctx).textContent; + assert.match(text, /state row could not be read on node-a/); + assert.match(text, /unknown, not idle/); +}); + +test('a cluster merge names the nodes whose state was unreadable, and flags them in the node table', async () => { + const status = { + ...STATUS, + node: null, + scope: 'cluster', + sources: { mode: 'merged', complete: true, configured: 2, answered: 2 }, + stateUnavailableOn: ['node-b'], + byNode: [ + { + hostname: 'node-a', + enabled: true, + dryRun: true, + stateAvailable: true, + sweepRunning: false, + canaryRunning: false, + }, + { + hostname: 'node-b', + enabled: true, + dryRun: true, + stateAvailable: false, + sweepRunning: false, + canaryRunning: false, + }, + ], + }; + const ctx = await ready({ status }); + const text = draw(ctx).textContent; + assert.match(text, /could not be read on node-b/); + assert.match(text, /state unreadable/); +}); + +test('a plugin without the state row is not accused of one it cannot read', async () => { + const ctx = await ready(); + assert.doesNotMatch(draw(ctx).textContent, /could not be read/); +}); + +test('a running sweep says how far it has got, at node scope and summed under the cluster merge', async () => { + const node = await ready({ + status: { ...STATUS, sweep: { ...STATUS.sweep, running: true, progress: { examinedApprox: 24_000 } } }, + }); + assert.match(draw(node).textContent, /sweep running · ~24,000 rows examined/); + + const cluster = await ready({ + status: { + ...STATUS, + node: null, + scope: 'cluster', + sources: { mode: 'merged', complete: true, configured: 2, answered: 2 }, + sweep: { + ...STATUS.sweep, + running: true, + runningOn: ['node-a', 'node-b'], + progress: [ + { hostname: 'node-a', examinedApprox: 10_000 }, + { hostname: 'node-b', examinedApprox: 4000 }, + ], + }, + byNode: [ + { + hostname: 'node-a', + enabled: true, + dryRun: true, + sweepRunning: true, + sweepProgress: { examinedApprox: 10_000 }, + }, + { + hostname: 'node-b', + enabled: true, + dryRun: true, + sweepRunning: true, + sweepProgress: { examinedApprox: 4000 }, + }, + ], + }, + }); + const text = draw(cluster).textContent; + // Disjoint slices, so the header sums; the node table keeps each node's own count. + assert.match(text, /~14,000 rows examined/); + assert.match(text, /sweeping · ~10,000 examined/); + assert.match(text, /sweeping · ~4,000 examined/); +}); + +test('a running sweep with no heartbeat count yet still reads as running, without a made-up number', async () => { + const ctx = await ready({ status: { ...STATUS, sweep: { ...STATUS.sweep, running: true, progress: null } } }); + const text = draw(ctx).textContent; + assert.match(text, /sweep running/); + assert.doesNotMatch(text, /rows examined/); +}); diff --git a/packages/console/test/trafficView.test.js b/packages/console/test/trafficView.test.js index 2c0df05d..f9110d06 100644 --- a/packages/console/test/trafficView.test.js +++ b/packages/console/test/trafficView.test.js @@ -28,7 +28,7 @@ installDom(); const { el } = await import('../src/admin/ui.js'); const { load, render, cadenceIndex, cadenceFor, notHitRows, originVerdict, originCostByReason, coverageSplit } = await import('../src/admin/views/traffic.js'); -const { ratioOf } = await import('../src/admin/charts.js'); +const { ratioOf, originLoad, fmtNet, isCacheServed } = await import('../src/admin/charts.js'); const HOUR = 3_600_000; const BUCKETS = 4; @@ -121,6 +121,13 @@ const ANALYTICS = { combo('response_500', 'p', 'GET', null, 60), // A served page whose age computed negative — dropped from the distribution above combo('prerender_ops', 'page_age_negative', 'googlebot', 'desktop', 3), + // What this system asked of the origin: 120 posted render results (one row each), one probe + // pass that made 60 requests (a VALUE — one emit carrying the count), one sitemap refresh + // that read 12 sitemaps. With 260 proxied serves the origin saw 452 of the 1,010 requests + // crawlers made: 74% gross offload, 55% net. + combo('render', 'outcome', 'rendered', null, 120), + combo('prerender_ops', 'probe_probed', null, null, 1, 60), + combo('prerender_ops', 'sitemap_sitemaps', null, null, 1, 12), ], }; @@ -409,6 +416,145 @@ test('the KPI strip separates a coverage miss from every other non-hit', async ( for (const family of ['Coverage', 'Cadence', 'Integrity']) assert.match(text, new RegExp(family)); }); +// ---- net offload ---------------------------------------------------------------- + +test('net offload subtracts every origin request this system made; gross subtracts only the proxied ones', () => { + const load = originLoad(ANALYTICS); + // 100 + 150 misses and 10 blob-timeouts, all answered from the origin. + assert.equal(load.proxied, 260); + // One row per posted result, so the emit COUNT is the render count … + assert.equal(load.renders, 120); + // … while the pass counters carry their count as a VALUE: one emit, 60 probes. sumCount here + // would report 1 and the origin load would be short by a whole pass. + assert.equal(load.probes, 60); + assert.equal(load.sitemaps, 12); + assert.equal(load.total, 452); + // The baseline is what ARRIVED (bot_request), not what was served. + assert.equal(load.arrived, 1010); + assert.ok(Math.abs(load.net - (1010 - 452) / 1010) < 1e-9); + // A pass counter contributed, so the figure is stamped as landing where a pass finished. + assert.equal(load.lumpy, true); + // The unmeasured term's exposure is every page handed to a crawler, not a subset of bots. + assert.equal(load.handed, 1010); +}); + +test('the KPI strip carries gross AND net, and says what the net figure leaves out', async () => { + const ctx = await ready(); + const text = textOf(ctx); + assert.match(text, /Origin offload · gross/); + assert.match(text, /74%/); + assert.match(text, /Origin offload · net/); + assert.match(text, /55%/); + // The fifth term is stated on the tile, on the panel's own tile, and in the note — never + // multiplied in as a guess. + assert.match(text, /before crawler follow-up requests/); + assert.match(text, /What the origin actually saw/); + assert.match(text, /Crawler follow-up requests/); + assert.match(text, /not measured/); + assert.match(text, /1,010 pages were handed to crawlers/); + // The omitted term is stated as two-sided: a saving where snapshots carry no scripts, a cost + // where they do — never as "the number would only be lower". + assert.match(text, /counted on neither side/); + assert.match(text, /true net offload for rendering crawlers is HIGHER than shown/); + // Every measured term is named with its count. + for (const term of ['proxied serves', 'renders', 'change probes', 'sitemap fetches']) + assert.match(text, new RegExp(term)); + // And the lumpiness caveat appears because a pass counter contributed. + assert.match(text, /land where a PASS FINISHED/); +}); + +test('a window with no crawler requests has no net offload — not a 100% one', () => { + const noBots = ANALYTICS.series.filter((s) => s.metric !== 'bot_request' && s.metric !== 'bot_serve'); + const load = originLoad({ ...ANALYTICS, series: noBots }); + assert.equal(load.arrived, 0); + assert.equal(load.net, null); + assert.equal(fmtNet(load.net), '—'); +}); + +test('serves stand in for arrivals only when the window has no bot_request rows at all', () => { + const noRequests = ANALYTICS.series.filter((s) => s.metric !== 'bot_request'); + assert.equal(originLoad({ ...ANALYTICS, series: noRequests }).arrived, 1010); +}); + +test('a fleet that out-requests its crawlers reads as a NEGATIVE offload, with the finding spelled out', async () => { + const analytics = { ...ANALYTICS, series: [...ANALYTICS.series, combo('render', 'outcome', 'rendered', null, 5000)] }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const figure = originLoad(analytics); + assert.ok(figure.net < 0); + // A signed percentage — pct() is for shares of a whole and would have nothing to divide by. + assert.match(fmtNet(figure.net), /^-\d+%$/); + const text = textOf(ctx); + assert.match(text, /The origin saw more requests than the crawlers made/); + // The gross figure is untouched by renders, which is the whole reason the net one exists. + assert.match(text, /74%/); +}); + +test('the net tile cannot narrow to one bot, and says so rather than showing a crawler’s number', async () => { + const ctx = await ready(); + ctx.data.bots = ['bingbot']; + const text = textOf(ctx); + assert.match(text, /before crawler follow-up requests · all bots/); + // Unchanged by the filter: renders, probes and sitemap fetches are not for any one crawler. + assert.match(text, /55%/); +}); + +test('nothing reaching the origin is 100% offload, and the follow-up caveat still stands', async () => { + const analytics = { + ...ANALYTICS, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 500), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 500), + ], + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /100% gross and net alike/); + assert.match(text, /500 pages were handed to crawlers/); +}); + +// ---- verified: a cache serve through an invalidation --------------------------- + +test('verified is a cache serve in the invalidation family — never "other", never outside cache-served', () => { + const rows = notHitRows([ + combo('bot_serve', 'cache', 'verified', 'googlebot', 40), + combo('bot_serve', 'origin', 'invalidated', 'googlebot', 60), + ]); + const byStatus = new Map(rows.map((row) => [row.status, row])); + // Same family: the population an invalidation touched, split into rescued and refused. + assert.equal(byStatus.get('verified').family, 'invalidated'); + assert.equal(byStatus.get('invalidated').family, 'invalidated'); + assert.deepEqual([...byStatus.get('verified').sources], [['cache', 40]]); + assert.ok(isCacheServed('verified')); + assert.ok(isCacheServed('peer-rescue')); + assert.ok(!isCacheServed('invalidated')); + assert.ok(!isCacheServed('blob-timeout')); +}); + +test('the per-route table counts a verified serve as cache-served', async () => { + const analytics = { + ...ANALYTICS, + series: [ + combo('bot_serve', 'cache', 'verified', 'googlebot', 50), + combo('bot_serve', 'cache', 'hit', 'googlebot', 50), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 100), + combo('route_serve', '/product/', 'verified', 'desktop', 50), + combo('route_serve', '/product/', 'hit', 'desktop', 50), + ], + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const row = rowFor(draw(ctx), '/product/'); + assert.ok(row, 'expected a /product/ row'); + // Columns: route, mode, serves, cache-served, … + assert.equal(row.children[3].textContent, '100%'); + // And the invalidation family is on the KPI strip with the verified share inside it. + const text = textOf(ctx); + assert.match(text, /Invalidation/); + assert.match(text, /rescued on evidence/); +}); + // ---- misses the origin cannot serve ----------------------------------------- test('originVerdict separates "nobody has this page" from "the origin is broken"', () => {