From f2cca27c1456cb70dfad84b316cc31a4542379f6 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 24 Sep 2026 17:04:30 -0400 Subject: [PATCH] feat(plugin): probe only sitemap-listed targets (opt-in), and re-render a product that rejoins its sitemap; v0.89.0 Three pieces, all off by default. Target.unlistedAt. A sitemap walk that unlinks a target (sitemapUrl -> null) now stamps it with THE START OF THAT WALK, in the same patch; the re-attach clears it in its own patch, and Target.suppress carries it through its whole-row put. No new writes. Stamping with the walk start rather than the prune's clock is what makes shear exact: a URL that moves to a later child of a paginated index is unlinked and re-attached inside one walk, so its stamp equals this walk's start and is not a rejoin; only a stamp from an earlier walk is. changeProbe.scope ('all' | 'listed', default 'all') and changeProbe.unlistedGrace (default 48h). Under 'listed' the sweep and the canary probe a target only while a sitemap lists it or for the grace after a walk unlinked it; never-listed (discovered) targets and long-gone products are skipped and counted as outOfScope in the pass stats. The check sits after the rule match and before `matched`, so the canary cohort and the continuous-mode slice size describe what is probed; the bootstrap cohort and every canary pass apply the same selection. A pass that skips more than it probes logs a warning, and scope 'listed' without an armed departure is a config warning. ingress.routes[].arrivalAction ('none' | 'render', default 'none') with a new sitemap.arrival group (enabled, dryRun default true, maxActions, maxCandidates; -1 = uncapped, as in #207). A rejoin is decided at re-attach time and acted on after the walk through the executor the departure check now shares with it (hard-expire past page.swrTtl, file the URL row at the current minute). Its own dryRun so arming departures never arms arrivals unmeasured. Tallied as `arrivals` on the result and progress row, not as metric series. Co-Authored-By: Claude Opus 5.5 (1M context) --- package-lock.json | 2 +- packages/plugin/METRICS.md | 10 + packages/plugin/README.md | 10 + packages/plugin/package.json | 2 +- packages/plugin/src/config.js | 30 +- packages/plugin/src/configSchema.js | 109 +++++ packages/plugin/src/resources/Sitemap.js | 175 +++++-- packages/plugin/src/resources/Target.js | 8 +- packages/plugin/src/schemas/schema.graphql | 9 + packages/plugin/src/util/changeProbe.js | 38 +- packages/plugin/src/util/probeScope.js | 60 +++ packages/plugin/src/util/routeClass.js | 43 +- packages/plugin/src/util/sitemapArrival.js | 118 +++++ packages/plugin/src/util/sitemapRun.js | 43 +- packages/plugin/src/util/time.js | 7 + packages/plugin/test/probeScope.test.js | 426 +++++++++++++++++ packages/plugin/test/sitemapArrival.test.js | 191 ++++++++ .../plugin/test/sitemapArrivalWalk.test.js | 430 ++++++++++++++++++ 18 files changed, 1659 insertions(+), 52 deletions(-) create mode 100644 packages/plugin/src/util/probeScope.js create mode 100644 packages/plugin/src/util/sitemapArrival.js create mode 100644 packages/plugin/test/probeScope.test.js create mode 100644 packages/plugin/test/sitemapArrival.test.js create mode 100644 packages/plugin/test/sitemapArrivalWalk.test.js diff --git a/package-lock.json b/package-lock.json index e74b118c..cfdefeb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.87.0", + "version": "0.89.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 8259b07d..ec413278 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -321,6 +321,16 @@ Grep-able, `[prerender]`-prefixed, and each is the richer record of something th "target with no schedule row" state. Any non-zero `restored` is worth an alert; nothing else reports it. - **`Sitemap refresh for finished: …`** — sitemaps processed, targets created/removed, duration. +- **`Arrival check: N rejoined …`** — the outcome tally of the post-walk sitemap-arrival action + (`ingress.routes[].arrivalAction`, `sitemap.arrival`): URLs a walk re-attributed after an EARLIER + walk unlinked them, with `render`/`would-render` and the named skips. Deliberately NOT a metric + series (a new `sitemap_*` family needs a console panel first); the same tally is `arrivals` on the + refresh result and on `GET /sitemap_refresh/`. +- **`change-probe sweep: changeProbe.scope left N rule-matched targets unprobed and probed only M`** — + under `changeProbe.scope: listed`, a pass that skipped more than it kept. Either the corpus really is + mostly unlisted (the setting is wrong for this site) or sitemap walks stopped listing what they used to; + check `sitemap_removed` and the departure tally. The per-pass count itself is `outOfScope` on the + `GET /prerender_admin/change-probe` pass record — the origin requests the scope saved. - **Render verdicts per URL** — `Suppressing prerendered url …`, `Prerender failed for … (reason)`, `Retrying … (failure strike N)`, `redirected to … which is `. The aggregate is the `render` metric's `outcome` series; these lines carry the URL. Most are `info`/`debug` now (they are normal diff --git a/packages/plugin/README.md b/packages/plugin/README.md index 349115a1..57552ef5 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -1291,6 +1291,16 @@ up as a `probe_failed` share and a loud log line, not as schedule churn. An extr path yields null is a failure too, so a shape change cannot flip every signature at once and mass-trigger. Suppressed targets are skipped — suppression owns its own recheck cadence. +**Scope** (`scope`, default `all`). `listed` probes only targets a sitemap lists now, plus those a walk +unlinked within `unlistedGrace` (48h by default, measured from `Target.unlistedAt`), and skips targets +discovered from traffic and products long gone from the sitemap — counted per pass as `outOfScope`, the +origin requests saved. It is only safe where the sitemap **is** the availability feed and the +sitemap-departure check is armed (`ingress.routes[].departureAction: render`, +`sitemap.departure.dryRun: false`), which already re-renders a product's page the day it leaves the +sitemap; a config warning says so otherwise. Its pair is `arrivalAction: render` on the same route +(`sitemap.arrival`, dry-run by default), which re-renders a product the day it **rejoins** its +sitemap instead of leaving its out-of-stock snapshot serving until the next cadence render. + Once a probe rule covers a route's volatile fields, that route's `renderInterval` can usually be raised substantially — the interval then only bounds what the probe cannot see (client-side content: reviews, image sets), and the render budget freed is what pays for the burst of diff --git a/packages/plugin/package.json b/packages/plugin/package.json index d6e6019e..a4ce7338 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.87.0", + "version": "0.89.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/config.js b/packages/plugin/src/config.js index 8c090754..f56e3b95 100644 --- a/packages/plugin/src/config.js +++ b/packages/plugin/src/config.js @@ -40,7 +40,7 @@ import { describeSecret } from './util/redact.js'; // evaluation time. The count has to come from the compiler rather than from raw config, // because the finding's whole job is to catch entries the compiler REJECTED (a typo'd // `match`), which the raw array still contains. -import { prerenderRouteCount } from './util/routeClass.js'; +import { DepartureAction, prerenderRouteCount } from './util/routeClass.js'; // NOT cyclic, on purpose: changeProbeSpec.js is the probe's pure half and imports nothing of the // runtime, precisely so this module can compile prospective rules inside collectConfigWarnings. import { inspectProbeRules } from './util/changeProbeSpec.js'; @@ -783,6 +783,34 @@ export const collectConfigWarnings = (target = config, { prerenderRoutes } = {}) 'nothing is re-rendered and nothing is invalidated until dryRun is turned off' ); } + // `scope: listed` stops probing a product once it has been out of its sitemap for `unlistedGrace`, + // on the premise that the departure check already re-rendered it on the way out. Without an armed + // departure nothing does, and the page keeps whatever it said when it left. Read off the RAW route + // list, not the compiled one: this may be checking a prospective config (see `prerenderRoutes`). + if (target.changeProbe.scope === 'listed') { + const { departure } = target.sitemap; + const routes = Array.isArray(target.ingress.routes) ? target.ingress.routes : []; + const routeDeparts = routes.some( + (route) => + route && + route.mode !== 'passthrough' && + (route.departureAction === DepartureAction.RENDER || route.departureAction === DepartureAction.EXPIRE) + ); + if (!departure.enabled || departure.dryRun || !routeDeparts) { + add( + 'warn', + 'changeProbe.scope', + "changeProbe.scope is 'listed' but sitemap departure is not armed (" + + (!routeDeparts + ? 'no prerender route sets departureAction' + : !departure.enabled + ? 'sitemap.departure.enabled is false' + : 'sitemap.departure.dryRun is true') + + ') — a product that leaves its sitemap stops being probed after changeProbe.unlistedGrace and ' + + 'nothing re-renders it on the way out, so its page keeps its last snapshot until its cadence render' + ); + } + } // The canary's bulk action needs the invalidation read path on; the mismatch would otherwise // surface only at trip time, which is 4am during the exact event the operator configured for. if ( diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 41b584f5..68ba82bf 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -105,6 +105,7 @@ export const configSchema = group('Prerender plugin configuration.', { 'Ordered route list (forwarded mode). Each entry is ' + "{ match: 'exact' | 'prefix' | 'contains', path: string, mode?: 'prerender' | 'passthrough', " + 'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean, demandFloor?: number, ' + + "departureAction?: 'none' | 'expire' | 'render', arrivalAction?: 'none' | 'render', " + 'rawCache?: boolean }.\n\n' + 'FIRST MATCH WINS, so order most-specific first. That ordering is what lets a passthrough ' + 'carve-out sit inside a prerendered prefix (`/products/clearance/` above `/products/`) ' + @@ -159,6 +160,19 @@ export const configSchema = group('Prerender plugin configuration.', { 'sitemap says something about that product, while a listing URL leaving a paginated sitemap ' + 'usually means the catalog was re-bucketed, and acting on it would expire current pages. ' + 'Bounded and observable by `sitemap.departure`.\n\n' + + "`arrivalAction` (default 'none', prerender routes only) — the other half: what happens when a " + + 'URL on this route REJOINS a sitemap after an earlier walk unlinked it. On the same retail ' + + 'catalog a product comes back to the sitemap the day it is restocked, and its cached page — ' + + 'last rendered while it was unavailable — keeps serving out-of-stock markup until its next ' + + 'cadence render.\n' + + ' none — re-attribute and nothing else, as before.\n' + + ' render — hard-expire the cached pages and file the URL to render at the current minute: ' + + 'the departure `render` action, through the same code.\n' + + 'A rejoin is told from a paginated sitemap\u2019s boundary shear (a URL moving to a later child ' + + 'is unlinked and re-attached inside ONE walk) by `Target.unlistedAt`, which the unlinking walk ' + + 'stamps with its own start — only a stamp from an earlier walk counts. Targets unlinked before ' + + 'v0.89.0 carry no stamp, so their rejoin is not seen. Bounded and observable by ' + + '`sitemap.arrival`.\n\n' + '`discoverTargets` (default true, prerender routes only) — whether a bot visiting an UNKNOWN ' + 'URL on this route creates a target for it. Set false on routes whose URL space is ' + 'combinatorial (faceted navigation, filter/sort permutations): crawlers walking those links ' + @@ -1082,6 +1096,40 @@ export const configSchema = group('Prerender plugin configuration.', { 'holds the sustained rate to ratePerSecond whatever origin latency does.', { min: 1 } ), + scope: option( + 'all', + 'Which targets the sweep and the canary probe.\n\n' + + '"all" (default) probes every owned, rule-matched target, however it entered the registry.\n\n' + + '"listed" probes a target only while a sitemap lists it (`sitemapUrl` set) or for ' + + '`unlistedGrace` after a walk unlinked it (`Target.unlistedAt`). Targets no sitemap has listed ' + + '(discovered from traffic), targets unlinked longer ago than the grace, and targets unlinked ' + + 'before v0.89.0 (which carry no stamp) are skipped and counted as `outOfScope` in the pass ' + + 'stats — the origin requests saved per pass. The canary cohort is built from, and re-checked ' + + 'against, the same selection. Suppressed targets are skipped under either scope, as always.\n\n' + + 'ONLY SAFE WHERE THE SITEMAP IS THE AVAILABILITY FEED AND DEPARTURE IS ARMED. On a catalog ' + + 'whose product sitemap drops an item the day it sells out, the departure check ' + + "(`ingress.routes[].departureAction: 'render'` with `sitemap.departure.dryRun: false`) already " + + 'expires and re-renders that page, so probing the unlisted long tail buys little. Anywhere else ' + + 'this simply stops watching those pages: an unlisted page whose price or availability changes ' + + 'keeps its snapshot until its cadence render. A config warning names the case where no route ' + + 'has departure armed, and a pass that skips more rule-matched targets than it probes logs a ' + + 'warning (most of the watched corpus not being in any sitemap is either a site this setting is ' + + 'wrong for, or sitemaps that stopped listing what they used to).\n\n' + + 'A FAILED sitemap walk cannot shrink the scope — it unlinks nothing. A child sitemap that parses ' + + 'but is truncated can, once it has stayed truncated for longer than the grace. Switching to ' + + '"listed" in continuous or anchored mode paces its first cycle against the old, larger slice, ' + + 'so that cycle finishes early; the next one paces to the real slice.', + { enum: ['all', 'listed'] } + ), + unlistedGrace: option( + 2 * DAY, + 'SCOPE "listed" ONLY: how long after a sitemap walk unlinks a target it is still probed. A URL ' + + 'briefly missing from one walk (a truncated child sitemap, an origin rebuilding its sitemaps) ' + + 'keeps being watched while later walks decide whether it really left, and a real departure is ' + + 'still probed while its re-render lands. Measured from the start of the walk that unlinked it. ' + + '0 probes only currently-listed targets.', + { unit: 'ms', min: 0 } + ), reprobeAfter: option( 12 * HOUR, 'Skip a URL whose stored baseline is younger than this. What makes a sweep RESUMABLE: the ' + @@ -2119,6 +2167,67 @@ export const configSchema = group('Prerender plugin configuration.', { ), } ), + arrival: group( + 'What a refresh does about a URL that REJOINS a sitemap — re-attributed by a walk after an ' + + 'EARLIER walk unlinked it. The action is declared PER ROUTE (`ingress.routes[].arrivalAction`); ' + + 'this group bounds and observes it, and nothing here does anything until at least one route ' + + 'opts in. The mirror of `sitemap.departure`, with its own switches so arming one never arms ' + + 'the other.\n\n' + + 'SHEAR IS KEPT OUT BY A STAMP, NOT BY WAITING. A walk that unlinks a target stamps ' + + '`Target.unlistedAt` with the walk\u2019s own START, and the re-attach compares against the ' + + 'current walk\u2019s start: a URL that moved to a later child of a paginated index is unlinked ' + + 'and re-attached inside one walk, so its stamp is this walk\u2019s start and it is not a rejoin. ' + + 'Rejoins are therefore known the moment they are re-attached; they are still ACTED on after the ' + + 'walk, by the same executor as departures.\n\n' + + 'WHAT IT DOES NOT SEE: targets unlinked before v0.89.0 (no stamp — they re-render on their ' + + 'cadence, as before); rejoins on a `revalidate: true` walk (which files every listed URL due ' + + 'now anyway). A URL that moves between two ROOT sitemaps looks like a departure in the first ' + + 'root\u2019s walk and a rejoin in the second\u2019s — the departure check has the same limitation, ' + + 'and the cost is one render per such URL, inside the ceilings.\n\n' + + 'Reported as `arrivals` on the refresh result and the progress row (`considered`, `capped`, and ' + + 'an outcome tally: `render`, or `would-render` under `dryRun`, for actions; `route-opted-out`, ' + + '`suppressed`, `unlinked`, `target-gone` and `capped` for the candidates nothing happened to) and ' + + 'in the walk\u2019s log — not as a metric series.', + { + enabled: option( + true, + 'Master switch for the arrival action. Routes still have to opt in, so leaving this on costs ' + + 'nothing until one does; it exists so an operator can stop the behaviour during an incident ' + + 'without editing the route list.' + ), + dryRun: option( + true, + 'Decide and report, write nothing. SEPARATE from `sitemap.departure.dryRun` on purpose: a ' + + 'deployment that has already armed departures and then opts a route into arrivals must still ' + + 'get a measured walk first, because the number that matters — how many rejoins one walk ' + + 'produces, and in particular the burst when a truncated child sitemap recovers and every URL ' + + 'it had dropped rejoins at once — is not one anybody has until they have looked. Inheriting ' + + 'the departure switch would act on the very first walk.' + ), + // -1 = no ceiling on both, for the reasons `sitemap.departure` spells out; read through + // `departureLimit`. + maxActions: option( + 5000, + 'Ceiling on rejoined URLs ACTED ON in one walk, or -1 for no ceiling. Skipped candidates do not ' + + 'count against it.\n\n' + + 'WHAT IT GUARDS AGAINST: the rejoin burst. A child sitemap that fetched truncated in one walk ' + + 'unlinked everything it dropped; when it recovers, every one of those URLs rejoins in the same ' + + 'walk, and without a ceiling that would expire and re-render all of them at once.\n\n' + + 'OVERFLOW IS DROPPED FOR GOOD, like a capped departure: the re-attach has already cleared the ' + + 'stamp, so no later walk sees the URL as a rejoin. It keeps its target, attribution and ' + + 'cadence; it just never gets its route\u2019s arrival action. -1 removes the ceiling; 0 acts on ' + + 'nothing (every candidate is still decided and reported).', + { min: -1 } + ), + maxCandidates: option( + 50000, + 'Ceiling on rejoined URLs HELD for the post-walk action, or -1 for no ceiling. Bounds memory, as ' + + '`sitemap.departure.maxCandidates` does. Overflow is DROPPED FOR GOOD and reported as ' + + '`arrivals.capped`. -1 removes the ceiling; 0 collects nothing.', + { min: -1 } + ), + } + ), userAgent: option( 'HarperSitemapCrawler/1.0', 'User-Agent for Harper’s sitemap crawler fetch. Unlike the proxy fetch UAs, a sitemap fetch ' + diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js index f97a650b..0a9bd4a3 100644 --- a/packages/plugin/src/resources/Sitemap.js +++ b/packages/plugin/src/resources/Sitemap.js @@ -11,6 +11,7 @@ import { setImmediate } from 'node:timers/promises'; import { applyInBatches, collectFromScan } from '../util/scan.js'; import { conditionalValidatorFor } from '../util/sitemapConditional.js'; import { decideDeparture, DepartureAction, departureCandidateCap, departureLimit } from '../util/sitemapDeparture.js'; +import { arrivalCandidateCap, decideArrival, isRejoin } from '../util/sitemapArrival.js'; import { cacheKeysOf } from './Target.js'; import { writeSchedule } from '../util/renderSchedule.js'; import { resolveEffectiveInterval } from '../util/routeClass.js'; @@ -61,6 +62,8 @@ class Sitemap extends databases.sitemaps.Sitemap { // Zero unless the departure check is on AND some route opts in; Infinity when // `maxCandidates` is -1. See util/sitemapDeparture.js. departureCap: departureCandidateCap(), + // The same for rejoins, off `sitemap.arrival`. See util/sitemapArrival.js. + arrivalCap: arrivalCandidateCap(), }); const visited = new Set(); const queue = [{ url: rootSitemapUrl, parentUrl: null }]; @@ -119,6 +122,15 @@ class Sitemap extends databases.sitemaps.Sitemap { run.countDeparture('failed'); } + // Rejoins were already told from shear during the walk (`isRejoin` against `run.startedAt`), so + // nothing forces this after the walk except sharing the departure executor. Guarded the same way. + try { + await processArrivals(run); + } catch (e) { + logger.error(`[prerender] Arrival check for ${rootSitemapUrl} failed: ${describeError(e)}`); + run.countArrival('failed'); + } + try { await onProgress?.(run.snapshot()); } catch (e) { @@ -372,6 +384,7 @@ const progressFields = (snapshot) => ({ // The departure tally rides the progress row too, so a dry run is readable from // `GET /sitemap_refresh/` without waiting for the walk to return. departures: snapshot.departures, + arrivals: snapshot.arrivals, }); /** @@ -592,9 +605,14 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, run.addTruncatedScan(sitemapUrl, examined, departed.length); } + // `unlistedAt` rides the unlink patch — no extra write — and is THE WALK'S START, not this prune's + // clock: a URL this same walk re-attaches further on then carries exactly `run.startedAt`, which is + // how the re-attach below tells shear from a rejoin (util/sitemapArrival.js). The probe's + // `changeProbe.scope: listed` grace is measured from it too (util/probeScope.js). + const unlistedAt = new Date(run.startedAt); await applyInBatches({ items: departed, - apply: (target) => Target.patch(target.url, { sitemapUrl: null }), + apply: (target) => Target.patch(target.url, { sitemapUrl: null, unlistedAt }), }); run.addRemoved(departed); @@ -620,15 +638,20 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, if (++considered % config.scan.yieldEvery === 0) await setImmediate(); let action; + let rejoined = false; if (revalidate) { + // No point read, so no rejoin is detected — and none needs to be: this files every listed URL + // due now. The `put` below replaces the row, which clears any `unlistedAt` by construction. action = TargetAction.RENDER; } else if (canSkipLookup({ revalidate, knownKeys, key: cacheUrl })) { action = TargetAction.SKIP; } else { // Only reached for a URL the prune scan did not return: genuinely new, moved here - // from another sitemap, or missed because `knownKeys` was capped. Only `sitemapUrl` - // is needed, so don't materialize the whole record in a bulk loop. - action = actionForExisting(await Target.get({ id: cacheUrl, select: ['sitemapUrl'] }), sitemapUrl, visited); + // from another sitemap, or missed because `knownKeys` was capped. Only the attribution + // and the unlink stamp are needed, so don't materialize the whole record in a bulk loop. + const existing = await Target.get({ id: cacheUrl, select: ['sitemapUrl', 'unlistedAt'] }); + action = actionForExisting(existing, sitemapUrl, visited); + rejoined = action === TargetAction.REATTACH && isRejoin(existing, run.startedAt); } switch (action) { @@ -646,8 +669,13 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, // Attribution changed, the page did not. `patch` leaves the RenderSchedule rows // alone; `put` would recompute `getInitialRenderTime` and shove the next render // forward by a fresh jitter every pass. See util/sitemapRun.js. + // + // `unlistedAt: null` in the same patch: re-attributed is listed, whether this is a rejoin, a + // same-walk shear, or a move from another sitemap. Unconditional, so the stamp cannot outlive + // the attribution it describes. run.count('updated'); - inflight.push(Target.patch(cacheUrl, { sitemapUrl, renderInterval })); + if (rejoined) run.addArrival(cacheUrl); + inflight.push(Target.patch(cacheUrl, { sitemapUrl, renderInterval, unlistedAt: null })); break; case TargetAction.CREATE: { @@ -738,30 +766,19 @@ function getTtlFromChangeFreq(changefreq, { minTtl, defaultTtl }) { } /** - * The post-walk departure check: decide, and act on, the URLs this walk unlinked. + * The action path both post-walk checks share: re-read each candidate, decide it, and — inside the + * ceiling, and outside a dry run — hard-expire its cached pages and, for `render`, file it due now. * - * Runs once per walk, after every child, because a URL that shifted to a LATER child of a - * paginated index is pruned before the child that now claims it is reached — mid-walk it is - * indistinguishable from one that genuinely left. `decideDeparture` re-reads each candidate and - * drops anything that picked up an attribution in the meantime; see util/sitemapDeparture.js for - * why that guard is the load-bearing part. - * - * Every candidate is decided and counted, including the ones nothing happens to, so a dry run - * answers the question a deployment actually has before enabling this: how many URLs a real walk - * would touch, and how many of the departures are shear rather than departure. - * - * A candidate refused by `maxActions` is counted `capped` and is gone for good — the walk already - * unlinked it, so no later walk offers it again. `maxActions: -1` removes the ceiling. + * ONE executor rather than one per check, because the action is the same action: a departing product + * page and a rejoining one are both pages whose cached snapshot is known to disagree with what the + * origin now says about availability. What differs is only how a candidate is decided (`decide`), which + * ceiling and dry-run switch apply, and where the outcome is counted. * - * Exported for tests. + * `decide` answers `{ action, reason }` with `action` spelled as a `DepartureAction` — `ArrivalAction` + * reuses those strings by construction (see util/routeClass.js), which is what lets one comparison + * serve both. */ -export async function processDepartures(run) { - const urls = run.departureCandidates(); - if (!urls.length) return; - - const { dryRun } = config.sitemap.departure; - // Through `departureLimit`, never raw: -1 is "no ceiling", and `acted >= -1` would refuse all. - const maxActions = departureLimit(config.sitemap.departure.maxActions); +async function actOnWalkCandidates({ urls, decide, dryRun, maxActions, count }) { let acted = 0; await applyInBatches({ @@ -769,14 +786,15 @@ export async function processDepartures(run) { apply: async (url) => { const target = await Target.get({ id: url, - // `sitemapUrl` is the shear guard, `state` keeps suppressed targets out, and the two - // cadence fields are what `resolveEffectiveInterval` needs to file a rung-correct row. + // `sitemapUrl` is the shear guard (departures) and the still-listed check (arrivals), + // `state` keeps suppressed targets out, and the two cadence fields are what + // `resolveEffectiveInterval` needs to file a rung-correct row. select: ['url', 'sitemapUrl', 'state', 'renderInterval', 'demandInterval'], }); - const { action, reason } = decideDeparture({ url, target }); + const { action, reason } = decide({ url, target }); if (action === DepartureAction.NONE) { - run.countDeparture(reason); + count(reason); return; } @@ -784,21 +802,21 @@ export async function processDepartures(run) { // parallel, so a check that awaited anything before incrementing would let a whole batch // through a cap of one. if (acted >= maxActions) { - run.countDeparture('capped'); + count('capped'); return; } acted++; if (dryRun) { - run.countDeparture(`would-${action}`); + count(`would-${action}`); return; } // HARD-expired, past the stale-while-revalidate window rather than to `now`. A plain // `Date.now()` expiry leaves the page 'swr' (`util/pageFreshness.js`), i.e. still SERVING // for another `page.swrTtl` — and the swr window exists to smooth over a late re-render of - // content presumed still right, which is exactly what a departed product page is not. - // This matches `changeProbe.triggerRevalidate`, which backdates for the same reason; + // content presumed still right, which is exactly what a departed (or rejoined) product page + // is not. This matches `changeProbe.triggerRevalidate`, which backdates for the same reason; // `Target.revalidate` keeps the plain expiry deliberately, because an operator asking for // a re-render is not asserting the content is wrong. const hardExpiredAt = Date.now() - config.page.swrTtl; @@ -815,23 +833,61 @@ export async function processDepartures(run) { // owner's floor and is never claimed again. This is the Target.revalidate lesson. await writeSchedule(url, { nextRenderTime: currentMinuteMs(), - // Derived, never a literal `false`, even though `decideDeparture` has already proved - // this target has no attribution and so this can only evaluate false. The literal is - // what `test/queueFunnel.test.js` forbids outright, and for a good reason: the - // derivation stays correct if this guard ever moves, where a literal would silently - // un-flag a sitemap-listed URL the day someone loosened the check above it. + // Derived, never a literal. For a departure `decideDeparture` has already proved this + // target has no attribution, so it can only evaluate false; for an arrival + // `decideArrival` has proved it has one, so it can only evaluate true. The literal + // `false` is what `test/queueFunnel.test.js` forbids outright, and for a good reason: + // the derivation stays correct if either guard ever moves, where a literal would + // silently mis-flag a URL the day someone loosened the check above it. fromSitemap: !!target.sitemapUrl, effectiveInterval: resolveEffectiveInterval(url, target), }); } - run.countDeparture(action); + count(action); }, }); +} - const { considered, capped, outcomes } = run.snapshot().departures; - const summary = Object.entries(outcomes) +/** An outcome tally as one log fragment: "render 3, reattached 12". */ +const summarizeOutcomes = (outcomes) => + Object.entries(outcomes) .map(([name, count]) => `${name} ${count}`) .join(', '); + +/** + * The post-walk departure check: decide, and act on, the URLs this walk unlinked. + * + * Runs once per walk, after every child, because a URL that shifted to a LATER child of a + * paginated index is pruned before the child that now claims it is reached — mid-walk it is + * indistinguishable from one that genuinely left. `decideDeparture` re-reads each candidate and + * drops anything that picked up an attribution in the meantime; see util/sitemapDeparture.js for + * why that guard is the load-bearing part. + * + * Every candidate is decided and counted, including the ones nothing happens to, so a dry run + * answers the question a deployment actually has before enabling this: how many URLs a real walk + * would touch, and how many of the departures are shear rather than departure. + * + * A candidate refused by `maxActions` is counted `capped` and is gone for good — the walk already + * unlinked it, so no later walk offers it again. `maxActions: -1` removes the ceiling. + * + * Exported for tests. + */ +export async function processDepartures(run) { + const urls = run.departureCandidates(); + if (!urls.length) return; + + const { dryRun } = config.sitemap.departure; + await actOnWalkCandidates({ + urls, + decide: decideDeparture, + dryRun, + // Through `departureLimit`, never raw: -1 is "no ceiling", and `acted >= -1` would refuse all. + maxActions: departureLimit(config.sitemap.departure.maxActions), + count: (outcome) => run.countDeparture(outcome), + }); + + const { considered, capped, outcomes } = run.snapshot().departures; + const summary = summarizeOutcomes(outcomes); logger.info( `[prerender] Departure check: ${considered} candidates` + `${capped ? ' (CAPPED at maxCandidates — the departed URLs past it were never checked, and no later walk will offer them)' : ''}` + @@ -848,6 +904,41 @@ export async function processDepartures(run) { } } +/** + * The post-walk arrival action: act on the URLs this walk found REJOINING a sitemap. + * + * Which URLs those are was settled during the walk — `isRejoin` compared each re-attached target's + * `unlistedAt` against this walk's start, which is what keeps same-walk shear out — so this only + * re-reads each one, applies `decideArrival`, and hands it to the executor departures use, under + * `sitemap.arrival`'s own dry-run switch and ceilings. See util/sitemapArrival.js. + * + * Reported in the run tally and the progress row (`arrivals`) and in the log, NOT as metric series: + * the tally carries every outcome already, and a new `sitemap_*` family is a console change (its + * metric-coverage guard reads these emit sites) that belongs with a panel to show it on. + * + * Exported for tests. + */ +export async function processArrivals(run) { + const urls = run.arrivalCandidates(); + if (!urls.length) return; + + const { dryRun } = config.sitemap.arrival; + await actOnWalkCandidates({ + urls, + decide: decideArrival, + dryRun, + maxActions: departureLimit(config.sitemap.arrival.maxActions), + count: (outcome) => run.countArrival(outcome), + }); + + const { considered, capped, outcomes } = run.snapshot().arrivals; + logger.info( + `[prerender] Arrival check: ${considered} rejoined` + + `${capped ? ' (CAPPED at maxCandidates — the rejoins past it were never checked, and no later walk will offer them)' : ''}` + + `${dryRun ? ', DRY RUN' : ''} — ${summarizeOutcomes(outcomes) || 'nothing to do'}` + ); +} + async function fetchLatestSitemap(url, { ifModifiedSince = null } = {}) { // Route every Harper→origin sitemap fetch through the same edge as the render/origin-fetch // path: whenever a staging IP is configured, pin the TCP connection to it (Host/SNI stay the diff --git a/packages/plugin/src/resources/Target.js b/packages/plugin/src/resources/Target.js index e49edcaa..47092a04 100644 --- a/packages/plugin/src/resources/Target.js +++ b/packages/plugin/src/resources/Target.js @@ -204,7 +204,7 @@ export class Target extends TargetTable { static async suppress(url, { reason, statusCode } = {}) { const existing = await Target.get({ id: url, - select: ['strikes', 'renderInterval', 'sitemapUrl', 'schedulerNode'], + select: ['strikes', 'renderInterval', 'sitemapUrl', 'schedulerNode', 'unlistedAt'], }); const strikes = countedStrikes(existing?.strikes) + 1; @@ -230,9 +230,15 @@ export class Target extends TargetTable { return { deleted: true, strikes }; } + // A put REPLACES the row, so every field that describes the URL rather than the verdict is carried + // across by name. `unlistedAt` with `sitemapUrl`: the two describe one fact (when a walk unlinked + // it), and dropping the stamp here would turn a recently-unlisted target into a never-listed one — + // out of `changeProbe.scope: listed` the moment the verdict heals, and invisible to the arrival + // check if it rejoins its sitemap. await TargetTable.put(url, { url, sitemapUrl: existing?.sitemapUrl ?? null, + unlistedAt: existing?.unlistedAt ?? null, schedulerNode: existing?.schedulerNode ?? getResidencyByUrl(url), renderInterval: existing?.renderInterval ?? null, state: 'suppressed', diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index 9ee0d260..367073a6 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -121,6 +121,15 @@ type Target @table(database: "render_service") @export { # never been evaluated and starts from its route/stored interval. Written ONLY when the rung # actually changes, so a converged corpus pays no write — same discipline as `strikes`. demandInterval: Long + # When a sitemap walk last UNLINKED this target (`sitemapUrl -> null`): THE START OF THAT WALK, not + # the moment of the prune, so a URL the same walk re-attaches further on (a paginated sitemap's + # boundary shear) is told from one that rejoins in a later walk by an exact comparison — see + # util/sitemapArrival.js. Cleared in the patch that re-attributes the target. Absent on a target no + # walk has unlinked since v0.89.0: one discovered from traffic, or one unlinked before the upgrade. + # Read by `changeProbe.scope: listed` (an unlisted target is still probed for `unlistedGrace`) and by + # the sitemap arrival check. Both ride writes the walk already makes. NOT @indexed: only ever read + # off a row already in hand. + unlistedAt: Date } # The render queue: ONE ROW PER URL, due at `nextRenderTime`. A claim of a row is one job that diff --git a/packages/plugin/src/util/changeProbe.js b/packages/plugin/src/util/changeProbe.js index 85e8a943..4cd9283d 100644 --- a/packages/plugin/src/util/changeProbe.js +++ b/packages/plugin/src/util/changeProbe.js @@ -60,6 +60,7 @@ import { dispatcherFor, configuredStagingIp } from './upstream.js'; import { cacheKeysOf } from '../resources/Target.js'; import { writeVerification } from './pageVerification.js'; import { walkUrlRange } from './urlWalk.js'; +import { probeScopeFilter } from './probeScope.js'; import { batchPause, cycleRatePerSecond, pacedRate, stepBackoff } from './probePacer.js'; import { loopLagMonitorState, readLoopLagMs, startLoopLagMonitor, stopLoopLagMonitor } from './loopLag.js'; import { isPassRunning, publishProbeState, readProbeState } from './probeState.js'; @@ -85,9 +86,10 @@ const probeStateTable = () => databases.probe_state.ProbeState; const YIELD_EVERY = 200; // What every probe read of the registry projects — what matching and the trigger need -// (writeSchedule wants fromSitemap + the cadence). The stored signature is NOT here: it lives -// in the node-local ProbeState table (see schema.graphql), read per probed URL. -const TARGET_SELECT = ['url', 'sitemapUrl', 'renderInterval', 'demandInterval', 'state']; +// (writeSchedule wants fromSitemap + the cadence), plus `unlistedAt` for `changeProbe.scope` +// (util/probeScope.js). The stored signature is NOT here: it lives in the node-local ProbeState +// table (see schema.graphql), read per probed URL. +const TARGET_SELECT = ['url', 'sitemapUrl', 'renderInterval', 'demandInterval', 'state', 'unlistedAt']; // Compile + memoize the rule list, keyed on config identity — applyOptions rebuilds config from // defaults on every change, so a fresh array means a reload (the routeClass memo pattern). @@ -135,7 +137,8 @@ export const cohortCollector = (count) => { const newStats = () => ({ examined: 0, // rows scanned owned: 0, // rows this node owns - matched: 0, // owned rows a rule matched (suppressed excluded) + matched: 0, // owned rows a rule matched (suppressed and out-of-scope excluded) + outOfScope: 0, // rule-matched rows `changeProbe.scope` left unprobed — the origin requests it saved; 0 under 'all' probed: 0, // probes attempted = seeded + rebaselined + unchanged + changed + failed seeded: 0, // first observation stored, nothing compared rebaselined: 0, // baseline was taken under a DIFFERENT rule fingerprint: observation stored, nothing compared or triggered (a rule edit, not a content change) @@ -509,6 +512,9 @@ export const runProbePass = async ({ now = Date.now, isCanceled = () => false, collectCohort = null, + // `changeProbe.scope` as a row predicate (util/probeScope.js), or null for 'all' — null never calls + // anything per row, which is what keeps the default pass's counters exactly the pre-option ones. + inScope = null, ownershipChecked = false, onYield = () => yieldNow(), reprobeAfter = 0, @@ -819,6 +825,13 @@ export const runProbePass = async ({ } } if (!rule) continue; + // AFTER the rule match, so `outOfScope` counts only rows that would otherwise have cost a probe, + // and BEFORE `matched`, so the canary cohort and the continuous-mode slice size (both built from + // matched rows) describe what is actually probed. + if (inScope && !inScope(row)) { + stats.outOfScope++; + continue; + } stats.matched++; collectCohort?.(rule, row.url); batch.push({ row, rule }); @@ -881,6 +894,16 @@ const emitStats = (stats, kind) => { const logPass = (stats, kind, dryRun) => { const line = { kind, dryRun, ...stats }; + // A scope that keeps less than it skips is either a site whose corpus really is mostly unlisted (and + // then `scope: listed` is the wrong setting for it) or sitemaps that stopped listing what they used + // to. Neither should be discovered from the origin-request savings looking good. + if (stats.outOfScope > stats.matched) { + logger.warn( + `[prerender] change-probe ${kind}: changeProbe.scope left ${stats.outOfScope} rule-matched targets ` + + `unprobed and probed only ${stats.matched} — most of the watched corpus is not in any sitemap. ` + + `Check the sitemap walks (sitemap_removed, the departure tally) before trusting this scope.` + ); + } // >50% failures is the replatform signature: the endpoint or markup this rule was written // against has probably changed shape, and every failed probe is a page silently back on // interval-only freshness. @@ -1111,6 +1134,7 @@ export const runProbeSweepOnce = async ({ dryRun, label = null, reseed = false } // A pending reseed cancels too: the pass that must stand down for it is this one. isCanceled: () => !config.changeProbe.enabled || sweepInterrupt !== null, collectCohort: (rule, url) => collectors.get(rule.label).add(url), + inScope: probeScopeFilter(config.changeProbe), }); // A cancelled pass abandons what is still queued: those rows never had their baseline // written, so the next pass re-detects them. Otherwise wait the queue out — the pass is not @@ -1197,10 +1221,13 @@ const ensureCohorts = async () => { const rules = probeRules(); const count = Math.max(1, config.changeProbe.canary.count | 0); const next = new Map(rules.map((rule) => [rule.label, []])); + // The sweep's selection, so a bootstrap cohort samples what the sweep probes. + const inScope = probeScopeFilter(config.changeProbe); for await (const row of walkTargets(config.changeProbe.chunkSize, () => countProbe('unreadable'))) { if (!config.changeProbe.enabled) break; if (getResidencyByUrl(row.url) !== server.hostname) continue; if (row.state === 'suppressed' || !isCanaryCandidate(row.url)) continue; + if (inScope && !inScope(row)) continue; const match = rules.find((rule) => buildProbeRequest(rule, row.url)); if (!match) continue; const cohort = next.get(match.label); @@ -1337,6 +1364,9 @@ export const runProbeCanaryOnce = async ({ dryRun } = {}) => { // the mass-change detector between sweeps, which is the one thing it exists for. reprobeAfter: 0, isCanceled: () => !config.changeProbe.enabled, + // Re-checked per pass, not only when the cohort was built: a member whose grace ran out + // since then is skipped (and counted) exactly as the sweep would skip it. + inScope: probeScopeFilter(config.changeProbe), }); await canaryTriggers.drain(); stats.triggered = canaryTriggers.stats.triggered; diff --git a/packages/plugin/src/util/probeScope.js b/packages/plugin/src/util/probeScope.js new file mode 100644 index 00000000..69590369 --- /dev/null +++ b/packages/plugin/src/util/probeScope.js @@ -0,0 +1,60 @@ +import { dateColumnMs } from './time.js'; + +/** + * WHICH TARGETS THE CHANGE PROBE WATCHES (`changeProbe.scope`). + * + * The sweep walks every owned target and spends one origin request on each one a rule matches. On a + * catalog whose product sitemap is its availability feed, a large share of those are products that + * LEFT the sitemap — measured at ~25% of probed product targets on one deployment — and for those the + * sitemap-departure check (`departureAction: render`) has already expired and re-rendered the page on + * the way out. Probing that long tail every night buys little and costs the origin a request per URL. + * + * all — every owned, rule-matched target (the behaviour before this option existed). + * listed — only a target a sitemap lists now (`sitemapUrl`), or one a walk unlinked within + * `unlistedGrace` (`Target.unlistedAt`). Never-listed targets (discovered from traffic), + * targets unlinked longer ago, and targets unlinked before the stamp existed are skipped. + * + * The grace is what makes the transition safe rather than abrupt: a URL briefly missing from one walk + * (a truncated child sitemap, an origin rebuilding its sitemaps) keeps being watched while later walks + * decide whether it really left, and a real departure is still probed while its re-render lands. + * + * A FAILED walk cannot shrink the scope: the prune runs only after a child is fetched and parsed, a + * root that fails aborts before any prune, a failing child is skipped, and an empty urlset is never + * reconciled. What can is a child that parses but is TRUNCATED, and stays truncated past the grace — + * the same event `sitemap.departure.maxActions` guards against, and one that shows up first as a + * `sitemap_removed` spike and in the departure tally. + * + * Pure, and kept out of util/changeProbe.js on purpose: the selection is the whole feature on the probe + * side, and it is testable without the probe's I/O. + */ + +export const ProbeScope = Object.freeze({ ALL: 'all', LISTED: 'listed' }); + +/** + * Is this target inside the `listed` scope at `nowMs`? + * + * Listed wins outright, whatever the stamp says: a stale `unlistedAt` on an attributed row can only + * come from a write outside the walk, and the attribution is the fact that matters. An unreadable or + * absent stamp reads as never listed. A stamp in the FUTURE (a walk on a node whose clock runs ahead) + * reads as recent — the direction that keeps probing. A grace of 0 is exactly "listed only". + */ +export const inListedScope = (row, { unlistedGrace, nowMs }) => { + if (row?.sitemapUrl) return true; + if (!(unlistedGrace > 0)) return false; + const unlistedAt = dateColumnMs(row?.unlistedAt); + return Number.isFinite(unlistedAt) && nowMs - unlistedAt < unlistedGrace; +}; + +/** + * The row predicate a pass filters with, or `null` for `all`. + * + * `null` rather than a predicate that always answers true, so the default pass never calls anything per + * row and its counters are exactly the pre-option ones. The clock is read per row: a paced pass runs + * for hours, and a grace judged against its start would keep probing targets that aged out mid-pass. + * + * @param {{ scope?: string, unlistedGrace?: number }} options `config.changeProbe` + * @param {() => number} [now] + * @returns {((row: object) => boolean) | null} + */ +export const probeScopeFilter = ({ scope, unlistedGrace } = {}, now = Date.now) => + scope === ProbeScope.LISTED ? (row) => inListedScope(row, { unlistedGrace, nowMs: now() }) : null; diff --git a/packages/plugin/src/util/routeClass.js b/packages/plugin/src/util/routeClass.js index d9d78a93..d207be85 100644 --- a/packages/plugin/src/util/routeClass.js +++ b/packages/plugin/src/util/routeClass.js @@ -57,6 +57,21 @@ export const DepartureAction = { }; const VALID_DEPARTURE = new Set(Object.values(DepartureAction)); +/** + * What a route does about a URL that REJOINS a sitemap after an earlier walk unlinked it — see + * util/sitemapArrival.js. Spelled with the departure enum's own strings, not merely equal ones: one + * post-walk executor (`resources/Sitemap.js`) serves both checks, and it compares against those. + * No `expire`: a rejoining page is not known to be wrong in a way that is worth sending bots to the + * origin for until its cadence render — the only useful arrival action is to render it now. + */ +export const ArrivalAction = { + /** Re-attribute and nothing else: the behaviour every route had before this existed. */ + NONE: DepartureAction.NONE, + /** Hard-expire the cached pages and file the URL to render now — the departure `render` action. */ + RENDER: DepartureAction.RENDER, +}; +const VALID_ARRIVAL = new Set(Object.values(ArrivalAction)); + // The allowlist every non-prerender class resolves to: keep every query param, so a proxied // request reaches the origin with the query the visitor actually sent. (`['*']` still // canonicalizes — params are sorted, the fragment and a trailing slash are dropped — it just @@ -189,7 +204,26 @@ const compileEntry = (raw, source, warn) => { } } - // Optional per-route raw-document cache. Same drop-the-FIELD rule as the four above: a typo here + // Optional per-route sitemap-arrival action — `departureAction`'s mirror, validated the same way for + // the same reason: a typo costs the route its arrival check, never how the path is served. + let arrivalAction = ArrivalAction.NONE; + if (raw.arrivalAction !== undefined && raw.arrivalAction !== null) { + if (mode === PASSTHROUGH) { + warn( + `ignoring arrivalAction on passthrough route "${raw.match} ${raw.path}" — a passthrough route ` + + `owns no cached page to expire and no schedule to advance` + ); + } else if (VALID_ARRIVAL.has(raw.arrivalAction)) { + arrivalAction = raw.arrivalAction; + } else { + warn( + `ignoring arrivalAction on route "${raw.match} ${raw.path}" — expected one of ` + + `${[...VALID_ARRIVAL].join(', ')}, got ${String(raw.arrivalAction)}` + ); + } + } + + // Optional per-route raw-document cache. Same drop-the-FIELD rule as the fields above: a typo here // must not change how the path is SERVED. Note this is the ROUTE's opt-in only — `render.raw.enabled` // is the master switch and is checked at the call site, so a route can carry the flag through a // deployment where the feature is off and mean exactly nothing. @@ -216,6 +250,7 @@ const compileEntry = (raw, source, warn) => { discoverTargets, demandFloor, departureAction, + arrivalAction, rawCache, source, }; @@ -348,6 +383,12 @@ export const anyRouteDeparts = () => { return false; }; +/** `anyRouteDeparts` for the arrival check — what keeps its candidate list empty until a route opts in. */ +export const anyRouteArrives = () => { + for (const entry of getRoutes()) if (entry.arrivalAction !== ArrivalAction.NONE) return true; + return false; +}; + /** How many prerender routes are configured. Config validation uses this — see below. */ export const prerenderRouteCount = () => { let count = 0; diff --git a/packages/plugin/src/util/sitemapArrival.js b/packages/plugin/src/util/sitemapArrival.js new file mode 100644 index 00000000..69878c33 --- /dev/null +++ b/packages/plugin/src/util/sitemapArrival.js @@ -0,0 +1,118 @@ +import { config } from '../config.js'; +import { anyRouteArrives, ArrivalAction, classifyUrl, PRERENDER } from './routeClass.js'; +import { departureLimit } from './sitemapDeparture.js'; +import { dateColumnMs } from './time.js'; + +/** + * What a URL REJOINING a sitemap means, and what to do about it — the other half of + * util/sitemapDeparture.js. + * + * On a catalog whose product sitemap is its availability feed, a product leaves the sitemap the day + * it sells out and comes back the day it is restocked. The departure check already re-renders the + * page on the way out, so what is cached for it is the OUT-OF-STOCK snapshot. When the product + * rejoins, the walk re-attributes the target and nothing else notices: that snapshot keeps serving + * until the page's next cadence render, which on a slow cadence is most of a day of telling bots an + * available product is unavailable. `arrivalAction: render` on the route files it to render now. + * + * ── TELLING A REJOIN FROM SHEAR ────────────────────────────────────────────────────────────── + * + * A paginated corpus shears across child boundaries (see sitemapDeparture.js): a URL that moves to + * a LATER child is unlinked by the child it left and re-attached by the child that now claims it, + * inside ONE walk. Seen from the re-attach alone that is indistinguishable from a rejoin, and acting + * on it would render every URL that shifted forward. + * + * `Target.unlistedAt` is what tells them apart. The prune stamps it with THE START OF THE WALK doing + * the unlinking — not the prune's own clock — so a same-walk unlink carries exactly this walk's + * start, and "unlinked before this walk began" is an exact comparison rather than a race against the + * clock. Only a stamp from an EARLIER walk is a rejoin (`isRejoin`). That also means, unlike a + * departure, a rejoin is decided at re-attach time without waiting for the walk to finish — but it is + * still ACTED on after the walk, by the same executor as departures, so both checks share one set of + * caps, one dry-run shape and one tally. + * + * ── WHAT IT DOES NOT SEE ───────────────────────────────────────────────────────────────────── + * + * - Targets unlinked before the stamp existed (v0.89.0) carry none, so their rejoin is invisible + * and they re-render on their cadence exactly as before. A discovered target a sitemap lists for + * the first time carries none either, which is correct: it never left anything. + * - A URL that moves between two ROOT sitemaps walked in sequence looks like a departure in the + * first walk and a rejoin in the second — each root is its own walk. The departure check has the + * same limitation; the cost is one extra render per such URL, inside the caps. + * - A `revalidate: true` walk files every listed URL due now without reading the target, so it + * detects nothing — and needs to detect nothing. + * + * ── WHY A CAPPED ARRIVAL IS GONE FOR GOOD ───────────────────────────────────────────────────── + * + * The re-attach clears `unlistedAt` in the same patch that restores the attribution, so the next + * walk sees an ordinary listed target. A rejoin past either ceiling keeps its target, its attribution + * and its cadence; it just never gets its route's arrival action. Same shape, and the same reason, as + * a capped departure. + */ + +// Re-exported for the same reason sitemapDeparture.js re-exports its enum: one module to reason +// through, one definition (in routeClass.js, which validates the route field). +export { ArrivalAction }; + +/** + * How many rejoined URLs one walk may hold for the post-walk action. Zero unless the check is on AND + * some route opts in — so the walk's `addArrival` does nothing for every deployment that has not asked + * for this — and Infinity for `maxCandidates: -1`. Resolved once per walk. + */ +export const arrivalCandidateCap = () => + config.sitemap.arrival.enabled && anyRouteArrives() ? departureLimit(config.sitemap.arrival.maxCandidates) : 0; + +/** + * Is re-attaching this target a REJOIN — did an earlier walk unlink it? + * + * `existing` is the row the walk's point read returned (`sitemapUrl`, `unlistedAt`). A target that is + * still attributed somewhere is moving between sitemaps, not rejoining, whatever its stamp says; one + * with no stamp was never unlinked by a walk that recorded it. And a stamp equal to this walk's start + * is this walk's own prune — the shear case — which is why the comparison is strict. + * + * @param {{ sitemapUrl?: string|null, unlistedAt?: unknown }|null|undefined} existing + * @param {number} walkStartedAt epoch ms — the same value this walk's prune stamps + */ +export const isRejoin = (existing, walkStartedAt) => { + if (!existing || existing.sitemapUrl) return false; + const unlistedAt = dateColumnMs(existing.unlistedAt); + return Number.isFinite(unlistedAt) && unlistedAt < walkStartedAt; +}; + +/** + * The action a route declares for URLs that rejoin a sitemap. Prerender routes only, and the field is + * already normalized by `compileEntry` — see `departureActionFor`, which this mirrors. + */ +export const arrivalActionFor = (url) => { + if (!config.sitemap.arrival.enabled) return ArrivalAction.NONE; + + const { routeClass, entry } = classifyUrl(url); + if (routeClass !== PRERENDER || !entry) return ArrivalAction.NONE; + + return entry.arrivalAction ?? ArrivalAction.NONE; +}; + +/** + * Decide one rejoined candidate, given what the post-walk re-read found. Pure, like + * `decideDeparture`; every skip is a named reason so a dry run can say why. + * + * The rejoin itself was established during the walk (`isRejoin`) — by now the re-attach has cleared + * the stamp — so this only re-checks what could have changed since. + * + * @param {object} candidate + * @param {string} candidate.url + * @param {object|null} candidate.target the target as re-read AFTER the walk, or null if absent + * @returns {{ action: string, reason: string }} + */ +export const decideArrival = ({ url, target }) => { + // Retired between the re-attach and now — by the gone path, an operator, or a purge. + if (!target) return { action: ArrivalAction.NONE, reason: 'target-gone' }; + + // Unlinked again before the walk ended (a concurrent walk of another root, or an operator). It is + // not listed, so it is not an arrival any more. + if (!target.sitemapUrl) return { action: ArrivalAction.NONE, reason: 'unlinked' }; + + // Suppression owns a suppressed target's schedule — the same reason `decideDeparture` skips it. + if (target.state === 'suppressed') return { action: ArrivalAction.NONE, reason: 'suppressed' }; + + const action = arrivalActionFor(url); + return { action, reason: action === ArrivalAction.NONE ? 'route-opted-out' : 'rejoined' }; +}; diff --git a/packages/plugin/src/util/sitemapRun.js b/packages/plugin/src/util/sitemapRun.js index f2700bf4..221a5bd7 100644 --- a/packages/plugin/src/util/sitemapRun.js +++ b/packages/plugin/src/util/sitemapRun.js @@ -91,7 +91,13 @@ export const actionForExisting = (existingTarget, sitemapUrl, claimedThisWalk) = * cannot act on a partial set while claiming success, and the only caller was discarding it. * - `failed` exists at all. One unreachable child used to abort the entire walk. */ -export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, departureCap = 0 } = {}) => { +export const createRefreshRun = ({ + removedSampleCap = 20, + failedCap = 100, + departureCap = 0, + arrivalCap = 0, + startedAt = Date.now(), +} = {}) => { const totals = { created: 0, updated: 0, @@ -132,7 +138,20 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar // unlinked, so no later walk will offer it again — which is why `capped` is reported at all. const departure = { candidates: [], capped: false, outcomes: {} }; + // Rejoined URLs held for the post-walk arrival action (util/sitemapArrival.js). Unlike departures + // these are decided AT re-attach time — `startedAt` is what makes that exact — and only acted on + // after the walk, so both checks share one executor, one set of caps and one tally shape. Same + // bound, same `arrivalCap: 0` default, same Infinity for uncapped. + const arrival = { candidates: [], capped: false, outcomes: {} }; + return { + /** + * When this walk started (epoch ms). The prune stamps it as `Target.unlistedAt` and the re-attach + * compares against it, so "unlinked by THIS walk" (shear) and "unlinked by an earlier one" (a + * rejoin) are told apart by an exact comparison, never by two readings of a clock. + */ + startedAt, + count(key, by = 1) { totals[key] += by; }, @@ -183,6 +202,23 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar departure.outcomes[outcome] = (departure.outcomes[outcome] ?? 0) + 1; }, + /** A re-attached target an EARLIER walk unlinked — held for the post-walk arrival action. */ + addArrival(url) { + if (arrivalCap <= 0) return; + if (arrival.candidates.length < arrivalCap) arrival.candidates.push(url); + else arrival.capped = true; + }, + + /** The rejoined URLs to re-read once the walk has finished. */ + arrivalCandidates() { + return arrival.candidates; + }, + + /** One arrival outcome, by name — `countDeparture`'s mirror. */ + countArrival(outcome) { + arrival.outcomes[outcome] = (arrival.outcomes[outcome] ?? 0) + 1; + }, + /** A child sitemap threw. The walk continues; the failure is reported, not swallowed. */ addFailure(url, error) { if (failed.length < failedCap) failed.push({ url, error: describeError(error) }); @@ -210,6 +246,11 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar capped: departure.capped, outcomes: { ...departure.outcomes }, }, + arrivals: { + considered: arrival.candidates.length, + capped: arrival.capped, + outcomes: { ...arrival.outcomes }, + }, }; }, }; diff --git a/packages/plugin/src/util/time.js b/packages/plugin/src/util/time.js index 1e380a1c..c1195b8e 100644 --- a/packages/plugin/src/util/time.js +++ b/packages/plugin/src/util/time.js @@ -23,6 +23,13 @@ export const currentMinuteMs = (ts = Date.now()) => Math.floor(ts / MINUTE) * MI */ export const epochMsOf = (value) => (value || value === 0 ? new Date(value).getTime() : Number.NaN); +/** + * `epochMsOf`, safe against a BigInt — for a `Date` column read inside a walk, where a throw ends the + * walk. A Date column can surface as a BigInt, and `new Date(10n)` THROWS rather than coercing (the + * trap `readSignature` in util/changeProbe.js defends against inline). Absent or unparseable is NaN. + */ +export const dateColumnMs = (value) => epochMsOf(typeof value === 'bigint' ? Number(value) : value); + /** * A NUMERIC column as a number, or `NaN` when the column is absent — the guard every reader of * `nextRenderTime` (and of any other numeric column) needs before comparing it to anything. diff --git a/packages/plugin/test/probeScope.test.js b/packages/plugin/test/probeScope.test.js new file mode 100644 index 00000000..0d07822f --- /dev/null +++ b/packages/plugin/test/probeScope.test.js @@ -0,0 +1,426 @@ +/** + * `changeProbe.scope` — which targets the sweep and the canary spend an origin request on. + * + * The pure predicate is pinned first; then the selection inside `runProbePass`; then the REAL sweep and + * canary (`runProbeSweepOnce` / `runProbeCanaryOnce`), which are the only place the cohort wiring can be + * seen. Those probe a loopback HTTP server, so what the origin was asked for is observed directly + * rather than inferred from counters. + */ +import { test, before, after, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; + +import { inListedScope, probeScopeFilter, ProbeScope } from '../src/util/probeScope.js'; + +const HOUR = 60 * 60 * 1000; +const DAY = 24 * HOUR; +const NOW = Date.parse('2026-09-24T12:00:00Z'); +const GRACE = 2 * DAY; + +// ---- the predicate ---- + +test('scope all is no filter at all — null, so the default pass calls nothing per row', () => { + assert.equal(probeScopeFilter({ scope: ProbeScope.ALL, unlistedGrace: GRACE }), null); + assert.equal(probeScopeFilter({}), null, 'absent reads as the default'); +}); + +test('listed: a target a sitemap lists is in scope, whatever its stamp says', () => { + const opts = { unlistedGrace: GRACE, nowMs: NOW }; + assert.equal(inListedScope({ sitemapUrl: 'https://site.example.com/s.xml' }, opts), true); + assert.equal( + inListedScope({ sitemapUrl: 'https://site.example.com/s.xml', unlistedAt: new Date(NOW - 30 * DAY) }, opts), + true + ); +}); + +test('listed: an unlisted target is in scope INSIDE the grace and out of it after', () => { + const opts = { unlistedGrace: GRACE, nowMs: NOW }; + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW - HOUR) }, opts), true); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW - GRACE + 1) }, opts), true); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW - GRACE) }, opts), false, 'boundary'); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW - 3 * DAY) }, opts), false); +}); + +test('listed: never listed, an unreadable stamp, or grace 0 are out; BigInt and future stamps are handled', () => { + const opts = { unlistedGrace: GRACE, nowMs: NOW }; + assert.equal(inListedScope({ sitemapUrl: null }, opts), false, 'discovered, or unlinked before v0.89.0'); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: null }, opts), false); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: 'garbage' }, opts), false); + assert.equal(inListedScope({ sitemapUrl: null, unlistedAt: BigInt(NOW - HOUR) }, opts), true, 'no throw on BigInt'); + assert.equal( + inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW + HOUR) }, opts), + true, + 'clock skew keeps probing' + ); + assert.equal( + inListedScope({ sitemapUrl: null, unlistedAt: new Date(NOW) }, { unlistedGrace: 0, nowMs: NOW }), + false, + 'grace 0 is listed-only' + ); +}); + +test('the filter reads the clock per row, so a target that ages out mid-pass drops out mid-pass', () => { + let now = NOW; + const inScope = probeScopeFilter({ scope: 'listed', unlistedGrace: GRACE }, () => now); + const row = { sitemapUrl: null, unlistedAt: new Date(NOW - GRACE + HOUR) }; + assert.equal(inScope(row), true); + now += 2 * HOUR; + assert.equal(inScope(row), false); +}); + +// ---- the probe, with Harper and the origin faked ---- + +let changeProbe; +let applyOptions; +let collectConfigWarnings; +let fnv1a32; + +let origin; // loopback probe endpoint +let port; +const asked = []; // probe paths the origin received + +const sabs = new Map(); +const sharedBufferStub = { + getUserSharedBuffer: (key, buffer) => { + if (!sabs.has(key)) sabs.set(key, buffer); + return sabs.get(key); + }, + tryLock: () => true, + unlock() {}, +}; +let sharedRows = new Map(); +class SharedBufferFake { + static primaryStore = sharedBufferStub; + static async get(key) { + return sharedRows.get(key) ?? undefined; + } + static async put(key, value) { + sharedRows.set(key, value); + } +} +class NoopTable { + static async get() {} + static async put() {} + static async patch() {} + static async delete() {} + static search() { + return []; + } +} + +// The registry: rows keyed by url, searched the way util/urlWalk.js asks (one-sided url ranges, sorted). +let registry = new Map(); +const matches = (row, { attribute, comparator, value }) => + comparator === 'greater_than_equal' + ? row[attribute] >= value + : comparator === 'greater_than' + ? row[attribute] > value + : comparator === 'less_than' + ? row[attribute] < value + : row[attribute] === value; +class RegistryTable extends NoopTable { + static async get({ id }) { + const row = registry.get(id); + return row ? { ...row } : undefined; + } + static async *search({ conditions = [], sort, limit = Infinity }) { + let rows = [...registry.values()].filter((row) => conditions.every((c) => matches(row, c))); + rows.sort((a, b) => (a.url < b.url ? -1 : a.url > b.url ? 1 : 0)); + if (sort?.descending) rows.reverse(); + for (const row of rows.slice(0, limit)) yield { ...row }; + } +} + +const warns = []; + +before(async () => { + origin = createServer((req, res) => { + asked.push(req.url); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ price: 10 })); + }); + await new Promise((resolve) => origin.listen(0, '127.0.0.1', resolve)); + port = origin.address().port; +}); + +after(async () => { + await new Promise((resolve) => origin.close(resolve)); +}); + +beforeEach(async () => { + globalThis.server = { hostname: 'node-a', workerIndex: 0, nodes: [], config: { http: {} } }; + warns.length = 0; + globalThis.logger = { debug() {}, info() {}, warn: (m) => warns.push(String(m)), error() {}, notify() {} }; + globalThis.databases = { + coordination: { SharedBuffer: SharedBufferFake }, + probe_state: { ProbeState: NoopTable, RenderExpectation: NoopTable }, + render_service: { Target: RegistryTable }, + page_cache: { PrerenderedPage: NoopTable }, + render_schedule: { RenderSchedule: NoopTable }, + invalidation: { Invalidation: NoopTable }, + }; + sharedRows = new Map(); + registry = new Map(); + asked.length = 0; + changeProbe = await import('../src/util/changeProbe.js'); + ({ applyOptions, collectConfigWarnings } = await import('../src/config.js')); + ({ fnv1a32 } = await import('../src/util/hash.js')); + changeProbe.resetChangeProbeState(); +}); + +afterEach(() => { + applyOptions({ changeProbe: { enabled: false } }); + changeProbe.resetChangeProbeState(); +}); + +const rules = () => [ + { + label: 'pdp', + pathPattern: '^/product/prd-([^/]+)', + source: 'request', + request: { urlTemplate: `http://127.0.0.1:${port}/price/$1` }, + extract: ['price'], + }, +]; + +const configure = (changeProbe = {}) => + applyOptions({ + changeProbe: { + enabled: true, + rules: rules(), + ratePerSecond: 10_000, + concurrency: 8, + reprobeAfter: 0, + canary: { count: 10, interval: 0 }, + ...changeProbe, + }, + }); + +/** The product id a probe request named — what the origin was actually asked about. */ +const probedIds = () => asked.map((path) => path.replace('/price/', '')).sort(); + +/** + * Four shapes of target, one each. `ids` lets the canary tests pick URLs the bootstrap cohort stride + * accepts (fnv1a32 % 16 === 0). + */ +const seedCorpus = (ids = ['listed', 'recent', 'stale', 'never']) => { + const [listed, recent, stale, never] = ids; + const now = Date.now(); + const put = (id, row) => + registry.set(`https://site.example.com/product/prd-${id}`, { + url: `https://site.example.com/product/prd-${id}`, + renderInterval: null, + demandInterval: null, + state: null, + ...row, + }); + put(listed, { sitemapUrl: 'https://site.example.com/sitemap_product_1.xml' }); + put(recent, { sitemapUrl: null, unlistedAt: new Date(now - HOUR) }); + put(stale, { sitemapUrl: null, unlistedAt: new Date(now - 5 * DAY) }); + put(never, { sitemapUrl: null }); + return { listed, recent, stale, never }; +}; + +// ---- runProbePass: the selection itself ---- + +const runPass = async ({ rows, inScope, collectCohort }) => { + const { compileProbeRules } = await import('../src/util/changeProbeSpec.js'); + const probed = []; + const stats = await changeProbe.runProbePass({ + rows: (async function* () { + yield* rows; + })(), + rules: compileProbeRules(rules()), + ownerOf: (url) => (url.includes('elsewhere') ? 'node-b' : 'node-a'), + hostname: 'node-a', + probe: async (rule, url) => { + probed.push(url); + return '[1]'; + }, + read: async () => null, + write: async () => {}, + submitTrigger: async () => {}, + dryRun: true, + maxTriggers: 100, + concurrency: 4, + ratePerSecond: 10_000, + pause: async () => {}, + inScope, + collectCohort, + }); + return { stats, probed }; +}; + +const row = (id, extra = {}) => ({ + url: `https://site.example.com/product/prd-${id}`, + sitemapUrl: null, + renderInterval: null, + state: null, + ...extra, +}); + +const MIXED = () => [ + row('listed', { sitemapUrl: 'https://site.example.com/s.xml' }), + row('recent', { unlistedAt: new Date(Date.now() - HOUR) }), + row('stale', { unlistedAt: new Date(Date.now() - 5 * DAY) }), + row('never'), + row('suppressed', { sitemapUrl: 'https://site.example.com/s.xml', state: 'suppressed' }), + row('elsewhere', { sitemapUrl: 'https://site.example.com/s.xml' }), + { ...row('x'), url: 'https://site.example.com/catalog/shoes' }, // no rule +]; + +test('runProbePass under listed: probes listed + recently unlisted, counts the rest as outOfScope', async () => { + const cohort = []; + const { stats, probed } = await runPass({ + rows: MIXED(), + inScope: probeScopeFilter({ scope: 'listed', unlistedGrace: GRACE }), + collectCohort: (rule, url) => cohort.push(url), + }); + const ids = probed.map((url) => url.split('prd-')[1]).sort(); + assert.deepEqual(ids, ['listed', 'recent']); + assert.equal(stats.outOfScope, 2, 'stale + never — the origin requests saved'); + assert.equal(stats.matched, 2, 'matched is what is probed, so continuous pacing sizes the real slice'); + assert.equal(stats.owned, 6); + assert.deepEqual(cohort.map((url) => url.split('prd-')[1]).sort(), ['listed', 'recent'], 'the cohort follows'); +}); + +test('suppressed and unowned rows are skipped exactly as before and are NOT counted as out of scope', async () => { + const { stats } = await runPass({ + rows: [row('suppressed', { state: 'suppressed' }), row('elsewhere')], + inScope: probeScopeFilter({ scope: 'listed', unlistedGrace: GRACE }), + }); + assert.equal(stats.outOfScope, 0); + assert.equal(stats.probed, 0); +}); + +test('defaults: with no scope filter every owned, unsuppressed, matched row is probed and outOfScope stays 0', async () => { + const { stats, probed } = await runPass({ rows: MIXED(), inScope: null }); + assert.deepEqual(probed.map((url) => url.split('prd-')[1]).sort(), ['listed', 'never', 'recent', 'stale']); + assert.equal(stats.outOfScope, 0); + assert.equal(stats.matched, 4); +}); + +// ---- the real sweep ---- + +test('the sweep under scope listed asks the origin only about in-scope targets', async () => { + configure({ scope: 'listed', unlistedGrace: GRACE }); + const { listed, recent } = seedCorpus(); + const result = await changeProbe.runProbeSweepOnce({ label: 'scoped' }); + + assert.deepEqual(probedIds(), [listed, recent].sort()); + assert.equal(result.outOfScope, 2); + assert.equal(result.matched, 2); + assert.equal(result.examined, 4); +}); + +test('the sweep under the default scope asks about every target, as before', async () => { + configure(); + seedCorpus(); + const result = await changeProbe.runProbeSweepOnce({ label: 'default' }); + assert.deepEqual(probedIds(), ['listed', 'never', 'recent', 'stale']); + assert.equal(result.outOfScope, 0); + assert.ok(!warns.some((w) => w.includes('changeProbe.scope left'))); +}); + +test('a sweep that skips more than it probes says so', async () => { + configure({ scope: 'listed', unlistedGrace: GRACE }); + seedCorpus(); + registry.delete('https://site.example.com/product/prd-recent'); // 1 in scope, 2 out + await changeProbe.runProbeSweepOnce({ label: 'mostly-unlisted' }); + assert.ok( + warns.some((w) => w.includes('changeProbe.scope left 2 rule-matched targets') && w.includes('probed only 1')) + ); +}); + +test('the canary cohort a SWEEP builds holds only in-scope targets', async () => { + configure({ scope: 'listed', unlistedGrace: GRACE }); + const { listed, recent } = seedCorpus(); + await changeProbe.runProbeSweepOnce({ label: 'build' }); + + asked.length = 0; + const canary = await changeProbe.runProbeCanaryOnce(); + assert.deepEqual(probedIds(), [listed, recent].sort()); + assert.equal(canary.perRule[0].cohort, 2); +}); + +// The bootstrap cohort (no sweep yet) samples 1-in-16 by hash, so pick ids it accepts. +const strideIds = (n) => { + const ids = []; + for (let i = 0; ids.length < n; i++) { + if (fnv1a32(`https://site.example.com/product/prd-${i}`) % 16 === 0) ids.push(String(i)); + } + return ids; +}; + +test('the BOOTSTRAP canary cohort (no sweep yet) applies the same selection', async () => { + configure({ scope: 'listed', unlistedGrace: GRACE }); + const { listed, recent } = seedCorpus(strideIds(4)); + const canary = await changeProbe.runProbeCanaryOnce(); + assert.equal(canary.perRule[0].cohort, 2); + assert.deepEqual(probedIds(), [listed, recent].sort()); +}); + +test('a cohort member whose grace ran out since the cohort was built is skipped and counted', async () => { + configure({ scope: 'listed', unlistedGrace: GRACE }); + const { listed, recent } = seedCorpus(strideIds(4)); + await changeProbe.runProbeCanaryOnce(); // builds [listed, recent] + + registry.get(`https://site.example.com/product/prd-${recent}`).unlistedAt = new Date(Date.now() - 3 * DAY); + asked.length = 0; + const canary = await changeProbe.runProbeCanaryOnce(); + assert.deepEqual(probedIds(), [listed]); + assert.equal(canary.perRule[0].outOfScope, 1); +}); + +test('the default scope leaves the bootstrap cohort exactly as before', async () => { + configure(); + seedCorpus(strideIds(4)); + const canary = await changeProbe.runProbeCanaryOnce(); + assert.equal(canary.perRule[0].cohort, 4); + assert.equal(canary.perRule[0].outOfScope, 0); +}); + +// ---- the config warning ---- + +const scopeFinding = () => collectConfigWarnings().find((f) => f.key === 'changeProbe.scope'); + +test('scope listed without an armed departure is a config warning, naming why', () => { + applyOptions({ changeProbe: { enabled: true, rules: rules(), scope: 'listed' } }); + assert.match(scopeFinding()?.message ?? '', /no prerender route sets departureAction/); + + applyOptions({ + changeProbe: { enabled: true, rules: rules(), scope: 'listed' }, + sitemap: { departure: { enabled: true, dryRun: true } }, + ingress: { mode: 'forwarded', routes: [{ match: 'prefix', path: '/product/', departureAction: 'render' }] }, + }); + assert.match(scopeFinding()?.message ?? '', /sitemap\.departure\.dryRun is true/); + + applyOptions({ + changeProbe: { enabled: true, rules: rules(), scope: 'listed' }, + sitemap: { departure: { enabled: false, dryRun: false } }, + ingress: { mode: 'forwarded', routes: [{ match: 'prefix', path: '/product/', departureAction: 'render' }] }, + }); + assert.match(scopeFinding()?.message ?? '', /sitemap\.departure\.enabled is false/); +}); + +test('scope listed with departure armed, or the default scope, raises no warning', () => { + applyOptions({ + changeProbe: { enabled: true, rules: rules(), scope: 'listed' }, + sitemap: { departure: { enabled: true, dryRun: false } }, + ingress: { mode: 'forwarded', routes: [{ match: 'prefix', path: '/product/', departureAction: 'render' }] }, + }); + assert.equal(scopeFinding(), undefined); + + applyOptions({ changeProbe: { enabled: true, rules: rules() } }); + assert.equal(scopeFinding(), undefined); +}); + +test('an invalid scope or grace is rejected back to the default', async () => { + const { config } = await import('../src/config.js'); + applyOptions({ changeProbe: { scope: 'sitemap-only', unlistedGrace: -1 } }); + assert.equal(config.changeProbe.scope, 'all'); + assert.equal(config.changeProbe.unlistedGrace, 2 * DAY); + applyOptions({ changeProbe: { scope: 'listed', unlistedGrace: 0 } }); + assert.equal(config.changeProbe.scope, 'listed'); + assert.equal(config.changeProbe.unlistedGrace, 0); +}); diff --git a/packages/plugin/test/sitemapArrival.test.js b/packages/plugin/test/sitemapArrival.test.js new file mode 100644 index 00000000..fc8a53a2 --- /dev/null +++ b/packages/plugin/test/sitemapArrival.test.js @@ -0,0 +1,191 @@ +/** + * The sitemap ARRIVAL check's pure half: the route field, the rejoin test that keeps shear out, the + * post-walk decision table, and the candidate list. The walk and the executor are exercised for real + * against fake tables in sitemapArrivalWalk.test.js. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { applyOptions, config } from '../src/config.js'; +import { defaultConfig } from '../src/configSchema.js'; +import { createRefreshRun } from '../src/util/sitemapRun.js'; +import { + ArrivalAction, + arrivalActionFor, + arrivalCandidateCap, + decideArrival, + isRejoin, +} from '../src/util/sitemapArrival.js'; +import { anyRouteArrives, inspectRoutes, matchRoute } from '../src/util/routeClass.js'; + +globalThis.logger ??= { debug() {}, info() {}, warn() {}, error() {} }; + +const PRODUCT = 'https://site.example.com/product/prd-1'; +const CATALOG = 'https://site.example.com/catalog/shoes'; +const SITEMAP = 'https://site.example.com/sitemap_product_1.xml'; +const WALK_START = Date.parse('2026-09-20T06:00:00Z'); + +const setRoutes = ({ productAction = 'render', arrival = {} } = {}) => + applyOptions({ + sitemap: { arrival: { enabled: true, dryRun: true, ...arrival } }, + ingress: { + mode: 'forwarded', + routes: [ + { + match: 'prefix', + path: '/product/', + mode: 'prerender', + ...(productAction ? { arrivalAction: productAction } : {}), + }, + { match: 'prefix', path: '/catalog/', mode: 'prerender' }, + ], + }, + }); + +// ---- defaults: nothing changes until a route opts in ---- + +test('defaults: every route compiles to arrivalAction none, and the group ships dry-run with the departure ceilings', () => { + const { arrival } = defaultConfig().sitemap; + assert.deepEqual(arrival, { enabled: true, dryRun: true, maxActions: 5000, maxCandidates: 50000 }); + + applyOptions({ ingress: { mode: 'forwarded', routes: [{ match: 'prefix', path: '/product/' }] } }); + assert.equal(matchRoute('/product/prd-1').arrivalAction, ArrivalAction.NONE); + assert.equal(anyRouteArrives(), false); + assert.equal(arrivalCandidateCap(), 0, 'nothing opted in, so the walk collects nothing'); +}); + +// ---- the route field, validated exactly like departureAction ---- + +test('arrivalAction render is accepted on a prerender route and read back per URL', () => { + setRoutes(); + assert.equal(matchRoute('/product/prd-1').arrivalAction, ArrivalAction.RENDER); + assert.equal(arrivalActionFor(PRODUCT), ArrivalAction.RENDER); + assert.equal(arrivalActionFor(CATALOG), ArrivalAction.NONE, 'the catalog route declares nothing'); + assert.equal(anyRouteArrives(), true); +}); + +test('an unrecognised arrivalAction drops the FIELD with a warning, never the route', () => { + const { prerender, dropped, warnings } = inspectRoutes( + [{ match: 'prefix', path: '/product/', arrivalAction: 'expire' }], + [] + ); + assert.equal(prerender, 1, 'the route still prerenders'); + assert.equal(dropped, 0); + assert.ok(warnings.some((w) => w.includes('ignoring arrivalAction') && w.includes('got expire'))); + + setRoutes({ productAction: 'bogus' }); + assert.equal(arrivalActionFor(PRODUCT), ArrivalAction.NONE); + assert.equal(anyRouteArrives(), false, 'an invalid action must not switch collection on'); +}); + +test('a passthrough route refuses arrivalAction with a warning', () => { + const { passthrough, warnings } = inspectRoutes( + [{ match: 'prefix', path: '/help/', mode: 'passthrough', arrivalAction: 'render' }], + [] + ); + assert.equal(passthrough, 1); + assert.ok(warnings.some((w) => w.includes('ignoring arrivalAction on passthrough route'))); +}); + +test('the master switch turns every route off without editing the route list', () => { + setRoutes({ arrival: { enabled: false } }); + assert.equal(arrivalActionFor(PRODUCT), ArrivalAction.NONE); + assert.equal(arrivalCandidateCap(), 0); +}); + +test('arrivalAction and departureAction are independent fields', () => { + applyOptions({ + sitemap: { departure: { enabled: true }, arrival: { enabled: true } }, + ingress: { mode: 'forwarded', routes: [{ match: 'prefix', path: '/product/', departureAction: 'render' }] }, + }); + assert.equal(matchRoute('/product/prd-1').departureAction, 'render'); + assert.equal(anyRouteArrives(), false, 'arming departure does not arm arrival'); +}); + +// ---- the rejoin test: what keeps shear out ---- + +test('isRejoin: a stamp from an EARLIER walk on an unattributed target is a rejoin', () => { + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: new Date(WALK_START - 1) }, WALK_START), true); + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: WALK_START - 86_400_000 }, WALK_START), true, 'epoch ms'); + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: new Date(WALK_START - 5).toISOString() }, WALK_START), true); + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: BigInt(WALK_START - 5) }, WALK_START), true, 'BigInt'); +}); + +// THE shear guard: this walk's own prune stamps exactly `run.startedAt`, so equality is shear. +test('isRejoin: a stamp equal to THIS walk’s start is this walk’s own prune — shear, not a rejoin', () => { + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: new Date(WALK_START) }, WALK_START), false); +}); + +test('isRejoin: no stamp, an unreadable stamp, a still-attributed target, or no row are not rejoins', () => { + assert.equal(isRejoin({ sitemapUrl: null }, WALK_START), false, 'discovered, or unlinked before v0.89.0'); + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: null }, WALK_START), false); + assert.equal(isRejoin({ sitemapUrl: null, unlistedAt: 'not a date' }, WALK_START), false); + // Attributed to a sitemap this walk has not reached yet: moving between children, not rejoining — + // even with a (stale) stamp. + assert.equal(isRejoin({ sitemapUrl: SITEMAP, unlistedAt: new Date(WALK_START - 1) }, WALK_START), false); + assert.equal(isRejoin(null, WALK_START), false); + assert.equal(isRejoin(undefined, WALK_START), false); +}); + +// ---- the post-walk decision table ---- + +test('decideArrival: a still-listed, unsuppressed target on an opted-in route renders', () => { + setRoutes(); + assert.deepEqual(decideArrival({ url: PRODUCT, target: { url: PRODUCT, sitemapUrl: SITEMAP, state: null } }), { + action: ArrivalAction.RENDER, + reason: 'rejoined', + }); +}); + +test('decideArrival: every skip is named', () => { + setRoutes(); + const listed = { url: PRODUCT, sitemapUrl: SITEMAP, state: null }; + assert.equal(decideArrival({ url: PRODUCT, target: null }).reason, 'target-gone'); + assert.equal(decideArrival({ url: PRODUCT, target: { ...listed, sitemapUrl: null } }).reason, 'unlinked'); + assert.equal(decideArrival({ url: PRODUCT, target: { ...listed, state: 'suppressed' } }).reason, 'suppressed'); + assert.equal( + decideArrival({ url: CATALOG, target: { ...listed, url: CATALOG } }).reason, + 'route-opted-out', + 'a route without arrivalAction' + ); + for (const target of [null, { ...listed, sitemapUrl: null }, { ...listed, state: 'suppressed' }]) { + assert.equal(decideArrival({ url: PRODUCT, target }).action, ArrivalAction.NONE); + } +}); + +// ---- the candidate list ---- + +test('arrivalCandidateCap follows sitemap.arrival.maxCandidates, -1 is Infinity, 0 is 0', () => { + setRoutes({ arrival: { maxCandidates: 123 } }); + assert.equal(arrivalCandidateCap(), 123); + setRoutes({ arrival: { maxCandidates: -1 } }); + assert.equal(arrivalCandidateCap(), Infinity); + setRoutes({ arrival: { maxCandidates: 0 } }); + assert.equal(arrivalCandidateCap(), 0); + assert.equal(config.sitemap.arrival.maxCandidates, 0); +}); + +test('addArrival holds up to the cap, then reports capped — and holds nothing at cap 0', () => { + const run = createRefreshRun({ arrivalCap: 2 }); + for (let i = 0; i < 5; i++) run.addArrival(`${PRODUCT}-${i}`); + assert.deepEqual(run.arrivalCandidates(), [`${PRODUCT}-0`, `${PRODUCT}-1`]); + assert.deepEqual(run.snapshot().arrivals, { considered: 2, capped: true, outcomes: {} }); + + const off = createRefreshRun(); + off.addArrival(PRODUCT); + assert.deepEqual(off.snapshot().arrivals, { considered: 0, capped: false, outcomes: {} }); + + const uncapped = createRefreshRun({ arrivalCap: Infinity }); + for (let i = 0; i < 60_000; i++) uncapped.addArrival(`${PRODUCT}-${i}`); + assert.equal(uncapped.snapshot().arrivals.considered, 60_000); + assert.equal(uncapped.snapshot().arrivals.capped, false); +}); + +test('the run carries the walk start it was given, and the arrival tally is separate from departures', () => { + const run = createRefreshRun({ startedAt: WALK_START, arrivalCap: 10, departureCap: 10 }); + assert.equal(run.startedAt, WALK_START); + run.countArrival('render'); + run.countArrival('render'); + run.countDeparture('reattached'); + assert.deepEqual(run.snapshot().arrivals.outcomes, { render: 2 }); + assert.deepEqual(run.snapshot().departures.outcomes, { reattached: 1 }); +}); diff --git a/packages/plugin/test/sitemapArrivalWalk.test.js b/packages/plugin/test/sitemapArrivalWalk.test.js new file mode 100644 index 00000000..fc6841ae --- /dev/null +++ b/packages/plugin/test/sitemapArrivalWalk.test.js @@ -0,0 +1,430 @@ +/** + * `Target.unlistedAt` and the sitemap arrival check, driven through REAL walks (`Sitemap.refresh`) + * against in-memory tables and a faked origin. + * + * What is pinned here is what a helper-level test cannot see: that the unlink and the re-attach write + * the stamp in the patches the walk already makes; that a URL shifting to a later child inside ONE + * walk (paginated-sitemap shear) is unlinked, stamped, re-attached and cleared WITHOUT reading as a + * rejoin; that a URL returning in a LATER walk does; and that the executor, the ceilings and the dry + * run behave for arrivals exactly as they do for departures. + */ +import { test, before, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { setTimeout as sleep } from 'node:timers/promises'; + +const targets = new Map(); // url -> Target row +const pages = new Map(); // cacheKey -> PrerenderedPage row +const sitemapRows = new Map(); +const patches = []; // every Target.patch, in order +const tablePuts = []; // every raw Target put (the sitemap CREATE path and Target.suppress) +const schedulePuts = []; +let documents = {}; + +const project = (row, select) => { + if (!select) return { ...row }; + if (typeof select === 'string') return row[select]; + return Object.fromEntries(select.map((key) => [key, row[key]])); +}; + +let applyOptions, config, sitemaps, cacheKeysOf, Target; + +before(async () => { + globalThis.Resource = class {}; + globalThis.server = { + hostname: 'node-1', + workerIndex: 0, + nodes: [], + config: { http: { port: 9926 } }, + recordAnalytics() {}, + }; + globalThis.logger = { debug() {}, info() {}, warn() {}, error() {}, notify() {} }; + const sabs = new Map(); + globalThis.databases = { + render_service: { + Target: class { + static async get(arg) { + const id = typeof arg === 'object' ? arg.id : arg; + const row = targets.get(id); + return row ? project(row, arg?.select) : undefined; + } + static async patch(id, fields) { + patches.push({ id, ...fields }); + const row = targets.get(id); + if (!row) throw new Error(`patch of a missing target ${id}`); + Object.assign(row, fields); + } + // A put REPLACES the row, as Harper's does — which is what makes the suppress test mean something. + static async put(id, data) { + tablePuts.push({ id, data }); + targets.set(id, { ...data, url: id }); + } + static async delete(id) { + targets.delete(id); + } + static search({ select, conditions = [] }) { + const rows = [...targets.values()].filter((row) => + conditions.every((condition) => row[condition.attribute] === condition.value) + ); + return (async function* () { + for (const row of rows) yield project(row, select); + })(); + } + }, + QueueControl: class {}, + }, + render_schedule: { + RenderSchedule: class { + static async put(key, row) { + schedulePuts.push({ key, ...row }); + } + static async delete() {} + }, + }, + page_cache: { + PrerenderedPage: class { + static async get({ id }) { + return pages.has(id) ? { cacheKey: id, ...pages.get(id) } : undefined; + } + static async patch(id, fields) { + Object.assign(pages.get(id), fields); + } + static async delete(id) { + pages.delete(id); + } + }, + }, + probe_state: { + ProbeState: class { + static async delete() {} + }, + RenderExpectation: class { + static async delete() {} + }, + }, + sitemaps: { + Sitemap: class { + static async get({ id, select }) { + const row = sitemapRows.get(id); + return row ? project(row, select) : undefined; + } + static async put(id, row) { + sitemapRows.set(id, { ...row, url: id }); + } + static search() { + return (async function* () {})(); + } + }, + SitemapRefresh: class {}, + }, + coordination: { + SharedBuffer: class { + static primaryStore = { + getUserSharedBuffer: (key, buffer) => { + if (!sabs.has(key)) sabs.set(key, buffer); + return sabs.get(key); + }, + tryLock: () => true, + unlock() {}, + }; + }, + }, + }; + globalThis.fetch = async (url) => { + const xml = documents[url]; + return xml === undefined + ? new Response('missing', { status: 404, statusText: 'Not Found' }) + : new Response(xml, { status: 200 }); + }; + + ({ applyOptions, config } = await import('../src/config.js')); + ({ sitemaps } = await import('../src/resources/Sitemap.js')); + ({ cacheKeysOf, Target } = await import('../src/resources/Target.js')); +}); + +const ROOT = 'https://site.example.com/sitemap_index.xml'; +const C1 = 'https://site.example.com/sitemap_product_1.xml'; +const C2 = 'https://site.example.com/sitemap_product_2.xml'; +const P = (n) => `https://site.example.com/product/prd-${n}`; +const CATALOG = 'https://site.example.com/catalog/shoes'; + +const index = (...children) => + `${children.map((c) => `${c}`).join('')}`; +const urlset = (...urls) => + `${urls.map((u) => `${u}`).join('')}`; + +/** Publish the two children's contents, then walk. */ +const walk = async ({ c1, c2, revalidate = false }) => { + documents = { [ROOT]: index(C1, C2), [C1]: urlset(...c1), [C2]: urlset(...c2) }; + return sitemaps.refresh(ROOT, { revalidate }); +}; + +const configure = ({ arrival = {}, productArrival = 'render', catalogArrival = null } = {}) => + applyOptions({ + sitemap: { + departure: { enabled: true, dryRun: true }, + arrival: { enabled: true, dryRun: true, ...arrival }, + }, + ingress: { + mode: 'forwarded', + routes: [ + { + match: 'prefix', + path: '/product/', + departureAction: 'render', + ...(productArrival ? { arrivalAction: productArrival } : {}), + }, + { match: 'prefix', path: '/catalog/', ...(catalogArrival ? { arrivalAction: catalogArrival } : {}) }, + ], + }, + }); + +const stampOf = (url) => targets.get(url)?.unlistedAt; + +beforeEach(() => { + targets.clear(); + pages.clear(); + sitemapRows.clear(); + patches.length = 0; + tablePuts.length = 0; + schedulePuts.length = 0; + configure(); +}); + +// ---- the stamp ---- + +test('an unlink stamps unlistedAt with the WALK’s start, in the same patch that clears sitemapUrl', async () => { + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + assert.equal(stampOf(P(1)), undefined, 'a freshly created target carries no stamp'); + + patches.length = 0; + const before = Date.now(); + await walk({ c1: [P(2)], c2: [P(3)] }); + const after = Date.now(); + + const unlink = patches.find((p) => p.id === P(1)); + assert.deepEqual(Object.keys(unlink).sort(), ['id', 'sitemapUrl', 'unlistedAt'], 'one patch, no extra write'); + assert.equal(unlink.sitemapUrl, null); + assert.ok(unlink.unlistedAt instanceof Date); + assert.ok(unlink.unlistedAt.getTime() >= before && unlink.unlistedAt.getTime() <= after); + assert.equal(targets.get(P(1)).sitemapUrl, null); + assert.equal(stampOf(P(2)), undefined, 'targets still listed are not touched'); +}); + +test('a re-attach clears the stamp in the same patch that restores the attribution', async () => { + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); // P1 leaves + assert.ok(stampOf(P(1)) instanceof Date); + + await sleep(5); + patches.length = 0; + await walk({ c1: [P(2)], c2: [P(1), P(3)] }); // P1 returns, to another child + + const reattach = patches.find((p) => p.id === P(1)); + assert.equal(reattach.sitemapUrl, C2); + assert.equal(reattach.unlistedAt, null); + assert.equal(stampOf(P(1)), null); +}); + +// THE shear case: a URL that moves to a LATER child is pruned by the child it left before the child +// that now claims it is reached. Its stamp is this walk's own start, so it must not read as a rejoin. +test('SHEAR: a URL moving to a later child in ONE walk is stamped, re-attached and cleared — and is not a rejoin', async () => { + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + + patches.length = 0; + const result = await walk({ c1: [P(2)], c2: [P(1), P(3)] }); + + const writes = patches.filter((p) => p.id === P(1)); + assert.equal(writes.length, 2, 'unlinked by C1, then re-attached by C2'); + assert.equal(writes[0].sitemapUrl, null); + assert.ok(writes[0].unlistedAt instanceof Date, 'the prune stamped it'); + assert.equal(writes[1].sitemapUrl, C2); + assert.equal(writes[1].unlistedAt, null, 'and the re-attach cleared it'); + assert.equal(stampOf(P(1)), null); + + assert.equal(result.arrivals.considered, 0, 'shear is not a rejoin'); + assert.deepEqual(result.arrivals.outcomes, {}); + assert.equal(result.departures.outcomes.reattached, 1, 'and the departure check drops it as shear too'); +}); + +test('a URL moving to an EARLIER child is re-attached before its old child is pruned, and never stamped', async () => { + await walk({ c1: [P(1)], c2: [P(2), P(3)] }); + + patches.length = 0; + const result = await walk({ c1: [P(1), P(3)], c2: [P(2)] }); + + const writes = patches.filter((p) => p.id === P(3)); + assert.equal(writes.length, 1); + assert.equal(writes[0].sitemapUrl, C1); + assert.equal(writes[0].unlistedAt, null); + assert.equal(result.arrivals.considered, 0); +}); + +// ---- the arrival check ---- + +test('a genuine rejoin — unlinked by an EARLIER walk — is a candidate, and a dry run acts on nothing', async () => { + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); // P1 leaves + await sleep(5); // walks must start on different milliseconds for the stamp to predate the next one + + schedulePuts.length = 0; + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + + assert.equal(result.arrivals.considered, 1); + assert.deepEqual(result.arrivals.outcomes, { 'would-render': 1 }); + assert.equal(schedulePuts.length, 0, 'dry run writes no schedule'); + assert.equal(targets.get(P(1)).sitemapUrl, C1); + assert.equal(stampOf(P(1)), null); +}); + +test('an armed rejoin hard-expires the cached pages past swrTtl and files the URL due at the current minute', async () => { + configure({ arrival: { dryRun: false } }); + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); + for (const key of cacheKeysOf(P(1))) pages.set(key, { expiresAt: Date.now() + 86_400_000 }); + await sleep(5); + + schedulePuts.length = 0; + const minuteBefore = Math.floor(Date.now() / 60_000) * 60_000; + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + const now = Date.now(); + + assert.deepEqual(result.arrivals.outcomes, { render: 1 }); + for (const key of cacheKeysOf(P(1))) { + assert.ok( + pages.get(key).expiresAt <= now - config.page.swrTtl, + `${key} is past the swr window, not merely expired` + ); + } + const filed = schedulePuts.filter((row) => row.key === P(1)); + assert.equal(filed.length, 1, 'ONE schedule row, keyed by the URL'); + assert.equal(filed[0].fromSitemap, true, 'a rejoined URL is sitemap-listed'); + assert.ok(filed[0].nextRenderTime >= minuteBefore && filed[0].nextRenderTime <= now); + assert.equal(filed[0].effectiveInterval, config.render.defaultInterval); +}); + +test('a rejoin on a route that has not opted in is decided and counted, never acted on', async () => { + configure({ arrival: { dryRun: false } }); // product opts in, catalog does not + await walk({ c1: [CATALOG, P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); // the catalog URL leaves + await sleep(5); + + schedulePuts.length = 0; + const result = await walk({ c1: [CATALOG, P(2)], c2: [P(3)] }); + assert.deepEqual(result.arrivals.outcomes, { 'route-opted-out': 1 }); + assert.equal(schedulePuts.length, 0); +}); + +test('a suppressed target that rejoins is left to suppression', async () => { + configure({ arrival: { dryRun: false } }); + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); + targets.get(P(1)).state = 'suppressed'; + await sleep(5); + + schedulePuts.length = 0; + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + assert.deepEqual(result.arrivals.outcomes, { suppressed: 1 }); + assert.equal(schedulePuts.length, 0); +}); + +test('a discovered target a sitemap lists for the first time is re-attributed, not an arrival', async () => { + await walk({ c1: [P(2)], c2: [P(3)] }); + targets.set(P(9), { url: P(9), sitemapUrl: null }); // discovered from traffic: no stamp + + const result = await walk({ c1: [P(2), P(9)], c2: [P(3)] }); + assert.equal(targets.get(P(9)).sitemapUrl, C1); + assert.equal(result.arrivals.considered, 0); +}); + +test('ceilings: maxActions caps what is acted on, maxCandidates what is held, and -1 lifts both', async () => { + const rejoinThree = async () => { + targets.clear(); + await walk({ c1: [P(1), P(2), P(3), P(4)], c2: [P(5)] }); + await walk({ c1: [P(4)], c2: [P(5)] }); // P1..P3 leave + await sleep(5); + return walk({ c1: [P(1), P(2), P(3), P(4)], c2: [P(5)] }); + }; + + configure({ arrival: { maxActions: 1 } }); + let result = await rejoinThree(); + assert.deepEqual(result.arrivals.outcomes, { 'would-render': 1, 'capped': 2 }); + + configure({ arrival: { maxActions: 0 } }); + result = await rejoinThree(); + assert.deepEqual(result.arrivals.outcomes, { capped: 3 }, '0 acts on nothing, and still decides every candidate'); + + configure({ arrival: { maxCandidates: 2 } }); + result = await rejoinThree(); + assert.equal(result.arrivals.considered, 2); + assert.equal(result.arrivals.capped, true); + + configure({ arrival: { maxActions: -1, maxCandidates: -1 } }); + result = await rejoinThree(); + assert.deepEqual(result.arrivals.outcomes, { 'would-render': 3 }); + assert.equal(result.arrivals.capped, false); +}); + +test('the arrival dry run is its own switch: armed departures do not arm arrivals', async () => { + applyOptions({ + sitemap: { departure: { enabled: true, dryRun: false }, arrival: { enabled: true } }, + ingress: { + mode: 'forwarded', + routes: [{ match: 'prefix', path: '/product/', departureAction: 'render', arrivalAction: 'render' }], + }, + }); + assert.equal(config.sitemap.arrival.dryRun, true); + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); + await sleep(5); + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + assert.deepEqual(result.arrivals.outcomes, { 'would-render': 1 }); +}); + +test('a revalidate walk detects no rejoin, and its put clears the stamp by replacing the row', async () => { + configure({ arrival: { dryRun: false } }); + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); + await sleep(5); + + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)], revalidate: true }); + assert.equal(result.arrivals.considered, 0); + assert.equal(stampOf(P(1)), undefined); + assert.equal(targets.get(P(1)).sitemapUrl, C1); +}); + +// ---- defaults ---- + +test('defaults: with no arrivalAction anywhere, a rejoin collects nothing and files nothing', async () => { + configure({ productArrival: null }); + await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + await walk({ c1: [P(2)], c2: [P(3)] }); + await sleep(5); + + schedulePuts.length = 0; + const result = await walk({ c1: [P(1), P(2)], c2: [P(3)] }); + assert.deepEqual(result.arrivals, { considered: 0, capped: false, outcomes: {} }); + assert.equal(schedulePuts.length, 0); + // The stamp is recorded whatever the arrival config says — it is what a later `scope: listed` or + // `arrivalAction` needs history from — and it rides writes the walk makes anyway. + assert.equal(stampOf(P(1)), null); +}); + +// ---- the other writer of sitemapUrl: suppression's whole-row put ---- + +test('Target.suppress carries unlistedAt across its whole-row put, beside sitemapUrl', async () => { + const stamp = new Date(Date.parse('2026-09-20T06:00:00Z')); + targets.set(P(7), { url: P(7), sitemapUrl: null, unlistedAt: stamp, strikes: 0 }); + + const { deleted } = await Target.suppress(P(7), { reason: 'noindex' }); + assert.equal(deleted, false); + const put = tablePuts.find((p) => p.id === P(7)); + assert.equal(put.data.state, 'suppressed'); + assert.equal(put.data.unlistedAt, stamp, 'dropping it would make a recently-unlisted target read as never listed'); + assert.equal(targets.get(P(7)).unlistedAt, stamp); + + // A listed target has no stamp, and suppression must not invent one. + targets.set(P(8), { url: P(8), sitemapUrl: C1, strikes: 0 }); + await Target.suppress(P(8), { reason: 'noindex' }); + assert.equal(targets.get(P(8)).unlistedAt, null); + assert.equal(targets.get(P(8)).sitemapUrl, C1); +});