diff --git a/package-lock.json b/package-lock.json index db6d6836..7bb0c89d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.89.0", + "version": "0.90.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/console/test/adminAssets.test.js b/packages/console/test/adminAssets.test.js index 08d8850a..5c0c1b42 100644 --- a/packages/console/test/adminAssets.test.js +++ b/packages/console/test/adminAssets.test.js @@ -278,6 +278,15 @@ test('a metric the plugin emits is charted by the console, or waived with a reas ['render_readiness.shortfall', 'report-only window; read raw from /prerender_admin/analytics — panel is #189'], ['render_readiness.rebaseline', 'report-only window; read raw from /prerender_admin/analytics — panel is #189'], ['render_readiness.satisfied_ms', 'report-only window; read raw from /prerender_admin/analytics — panel is #189'], + // The entity discovery gate (plugin v0.90.0) ships in dry run. Its evaluation census is read raw + // from the analytics endpoint during that window; the refusals that matter once it is ARMED are + // also emitted as `discovery_gated` with the gate name `entity`, which the Discovery gate panel + // already totals — so no console release is needed to see the armed gate's effect. A tile for + // the census (would-gate / suppressed-only / no-siblings / no-prefix) is the follow-up. + [ + 'prerender_ops.entity_gate', + 'dry-run census; read raw from /prerender_admin/analytics — armed refusals land in discovery_gated', + ], ]); const client = [...clientSources.values()].join('\n'); diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 4f123f7e..649f36de 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -185,6 +185,18 @@ Notes that bite: distinguishes a real crawler from a spoofed User-Agent: bot names come from the UA string alone, with no reverse-DNS or IP verification, so cross-check a search engine's own reporting before treating a per-bot breadth figure as that engine's crawl rate. +- **The entity discovery gate is `prerender_ops` / `entity_gate`, and it is not in the table row + above** (v0.90.0). One emit per evaluation, detail = outcome, context = bot: `gated` (a sibling URL + of the same entity is in rotation; not minted), `would-gate` (the same verdict under + `ingress.entityGate.dryRun`; minted anyway), `suppressed-only` (every sibling suppressed; minted), + `no-siblings` (a new entity; minted), `no-prefix` (the route has an `entityPrefix` and this URL + produced no usable match; minted), `error` (the sibling read threw; minted). The outcomes sum to the + mints the gate evaluated. **`would-gate` is the dry-run number** — read it against `render` outcome + `suppressed`/`canonical-mismatch` before arming. A route that is nearly all `no-prefix` has a + pattern that does not match its URLs. Armed refusals are ALSO emitted as `discovery_gated` with the + gate name `entity` (never in a dry run), so every view of what the gates hold out includes them; + unlike the `route`/`bot` gates, `entity` is evaluated only for URLs with no target row, so it counts + refused mints rather than gated misses on known targets. - **`queue_health.overdue` includes in-flight renders** (a leased row keeps its past due time), so its healthy floor is the in-flight count, not zero — and it is not comparable with numbers from before v0.34.0. The scan is capped by `management.scanCap`: a backlog past the cap reports the cap. diff --git a/packages/plugin/README.md b/packages/plugin/README.md index 57552ef5..4406ad71 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -257,6 +257,61 @@ forwarding `/blog/*`"); passthrough is the coverage backlog ("we proxy this much on purpose"). The tally is in-process, so **every worker** flushes its own line — each carries `node=` and `worker=`, and a reader sums across them. +### Discovery: one target per entity (`entityPrefix`) + +Traffic discovery mints a target for any unknown URL a bot gets a cacheable 200 for. `Target` is +keyed by URL, so two spellings of one product are two unrelated rows — and crawlers keep asking for +old or invented slugs of products whose correct URL is already tracked. Each one is minted, renders, +finds its canonical pointing elsewhere, is suppressed as `canonical-mismatch`, re-renders on the +suppression recheck until `maxStrikes` deletes it, and is minted again by the next crawler hit. +Measured on one deployment: `canonical-mismatch` was 8.2% of all render outcomes, and 150 of 150 +sampled suppressed targets had another target for the same product id under the correct slug. + +A route that declares `entityPrefix` stops that at discovery: + +```yaml +ingress: + routes: + - { match: prefix, path: '/product/prd-', queryParams: [], entityPrefix: '^/product/prd-[^/]+/' } + entityGate: + dryRun: true # the default: evaluate and count, mint anyway +``` + +- **The prefix.** The regex is anchored at the start of the URL path; its match plus the URL's origin + is the entity prefix (`https://www.example.com/product/prd-123/`). Every target whose URL starts + with it is a **sibling**. +- **The match must end on `/`.** `…/prd-123` is a string prefix of `…/prd-1234/…`, so a prefix that + stops mid-segment would let product 1234 gate product 123 forever. A match that does not end in + `/` is ignored for that URL (it is discovered as before, counted `no-prefix`), and a pattern whose + source does not end in `/` is warned about at config time. +- **The decision.** A sibling **in rotation** (not suppressed) means the entity already has a target + that will render, so the URL is not minted — it is still served (a miss proxies the origin). If + there are no siblings, or every sibling is **suppressed**, the URL is minted. Suppressed siblings + never block, which is what makes a genuine slug change a bounded delay rather than a permanent + block: the new slug is gated until the old slug's next render suppresses it (one render cycle of + the old target), then minted on its next crawl. `sitemapUrl` plays no part — a sitemap that lists + a non-canonical URL holds a listed-but-suppressed row, and the canonical URL must still be + discoverable. +- **Discovery only.** Sitemap ingestion, redirect adoption and the REST API create targets without + consulting the gate; the declared corpus is always created. +- **The read.** One one-sided primary-key range over `Target` (at most 3 rows, projecting `url` and + `state`, stopping at the first sibling in rotation), only for a URL with no target row, inside the + detached discovery step — never on the response path. `Target` is not residency-pinned, so the read + is node-local. Any failure mints. The 3 is fixed: measured on a production catalog, the first + sibling in rotation was within the first 3 keys for 150/150 sampled entities; past it the gate + mints, never refuses. + +Every URL under one prefix is the same entity, query variants the route keeps included — don't set +`entityPrefix` on a route where several URLs per entity are distinct pages. The pattern runs against +crawler-supplied paths, so keep it linear: a literal prefix plus `[^/]+` segments, no nested +quantifiers. + +**Rollout.** Deploy with `dryRun: true` (the default), read `prerender_ops` / `entity_gate` outcome +`would-gate` against `render` outcome `suppressed`/`canonical-mismatch`, then set `dryRun: false`. +Existing suppressed rows are untouched: they age out through `maxStrikes` as before, and once the gate +is armed they are not re-minted. Armed refusals are also counted on `discovery_gated` with the gate +name `entity`. + ### Sitemaps are filtered to prerender routes A sitemap is written for search engines: it lists every indexable URL on the site, which is routinely diff --git a/packages/plugin/package.json b/packages/plugin/package.json index a4ce7338..6bd7a586 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.89.0", + "version": "0.90.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 77f44e93..4b49179d 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -106,7 +106,7 @@ export const configSchema = group('Prerender plugin configuration.', { "{ match: 'exact' | 'prefix' | 'contains', path: string, mode?: 'prerender' | 'passthrough', " + 'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean, demandFloor?: number, ' + "departureAction?: 'none' | 'expire' | 'render', arrivalAction?: 'none' | 'render', " + - 'rawCache?: boolean }.\n\n' + + 'rawCache?: boolean, entityPrefix?: string }.\n\n' + 'FIRST MATCH WINS, so order most-specific first. That ordering is what lets a passthrough ' + 'carve-out sit inside a prerendered prefix (`/products/clearance/` above `/products/`) ' + 'without a second list and a precedence rule.\n\n' + @@ -188,7 +188,25 @@ export const configSchema = group('Prerender plugin configuration.', { 'proxy, never a render: a stale or invalidated snapshot still proxies live. Pair it with ' + '`discoverTargets: false` — gated URLs are exactly the population this is for, and gating ' + 'without it leaves them missing on every request forever. Enable it only on a route whose ' + - 'server-rendered document already carries its SEO surface; see `render.raw`.', + 'server-rendered document already carries its SEO surface; see `render.raw`.\n\n' + + '`entityPrefix` (prerender routes only; off when absent) — a regular expression, anchored at the ' + + 'start of the URL PATH, whose match is the part of the URL that identifies ONE entity, e.g. ' + + "`'^/product/prd-[^/]+/'` on a `/product/prd-` route. Traffic discovery then refuses to mint a " + + 'target for an unknown URL when another URL of the same entity (a target whose URL starts with ' + + 'this URL\u2019s origin plus the match) is already in rotation: crawlers requesting an old or ' + + 'invented slug of a product whose correct URL is tracked no longer cost a render that ends ' + + 'suppressed as `canonical-mismatch`. The URL is still served (a miss proxies the origin). ' + + 'Sitemap-listed URLs are always created. Siblings that are all SUPPRESSED do not block, so a ' + + 'product whose slug really changed is discovered at its new URL once the old URL\u2019s next ' + + 'render has suppressed it — a delay of one render cycle of the old target, never a permanent ' + + 'block. THE MATCH MUST END ON `/`: a prefix that stops mid-segment is a string prefix of other ' + + "entities' URLs (`prd-123` of `prd-1234`), so a match that does not end in `/` is ignored for " + + 'that URL (it is discovered as if no entityPrefix were set) and a pattern whose source does not ' + + 'end in `/` is warned about. Every URL under one prefix counts as the same entity, query variants ' + + 'the route keeps included, so do not set this on a route where several URLs per entity are ' + + 'distinct pages. The pattern runs against crawler-supplied paths, so keep it linear — a literal ' + + 'prefix plus `[^/]+` segments, no nested quantifiers. Governed by `ingress.entityGate` (dry run ' + + 'by default).', { itemType: 'object' } ), discoveryBots: option( @@ -203,6 +221,39 @@ export const configSchema = group('Prerender plugin configuration.', { 'ingestion are all unaffected.', { itemType: 'string' } ), + entityGate: group( + 'The entity discovery gate. Nothing here does anything until a route declares ' + + '`ingress.routes[].entityPrefix` — see that field for what an entity is.\n\n' + + 'WHAT IT DOES. When a bot requests an unknown URL on such a route and discovery would mint a ' + + 'target for it, the gate first reads the targets that share its entity prefix (one bounded ' + + 'primary-key range read of at most 3 rows — fixed, not tunable: measured, the first sibling in ' + + 'rotation was within the first 3 keys for every sampled entity — detached from the response and ' + + 'node-local). If one of them is in rotation, the URL is not minted. If there are none, or every ' + + 'one is suppressed, it is minted exactly as before. Any failure mints.\n\n' + + 'WHAT IT SAVES. Each refused mint is a render (and an origin document fetch) that would have ' + + 'ended `suppressed/canonical-mismatch`, then re-rendered on the suppression recheck until ' + + '`maxStrikes` deleted it and the next crawler hit minted it again. Existing suppressed rows are ' + + 'left alone — they age out through `maxStrikes` as before, and once the gate is armed they are ' + + 'not re-minted.\n\n' + + 'Observed on `prerender_ops` / `entity_gate`, one emit per evaluation by outcome; armed ' + + 'refusals are also counted on `discovery_gated` with the gate name `entity`.', + { + enabled: option( + true, + 'Master switch. Routes still have to opt in with `entityPrefix`, so leaving this on costs ' + + 'nothing until one does; it exists so an operator can stop the gate during an incident ' + + 'without editing the route list. Off = no sibling read, nothing counted, every unknown URL ' + + 'minted.' + ), + dryRun: option( + true, + 'Evaluate and count, but mint anyway. The default, because the number to know first is how ' + + 'many mints the gate WOULD refuse — `entity_gate` outcome `would-gate` — which is read ' + + 'against `render` outcome `suppressed`/`canonical-mismatch` before arming it. Turn it off ' + + 'to arm the gate.' + ), + } + ), excludePathPatterns: option( ['/search/'], 'Paths never auto-scheduled for rendering. Compiled into `routes` as ' + diff --git a/packages/plugin/src/http_handlers/bot_request.js b/packages/plugin/src/http_handlers/bot_request.js index 6846f5e9..26ccfd35 100644 --- a/packages/plugin/src/http_handlers/bot_request.js +++ b/packages/plugin/src/http_handlers/bot_request.js @@ -34,6 +34,7 @@ import { recordVisit } from '../util/visitFilter.js'; import { materializeCachedBody } from '../util/cachedBody.js'; import { captureForRawCache, rawCachePolicy, rawKeyOf, readRawPage } from '../util/rawCache.js'; import { rescueFromOwner } from '../util/peerRescue.js'; +import { evaluateEntityGate } from '../util/entityGate.js'; import { deliverResource } from './response.js'; export async function handleBotRequest(request) { @@ -443,6 +444,11 @@ async function resolveResource({ request, url, cacheUrl, deviceType, routeClass, // Target.get entirely, which matters on a gated combinatorial route where misses are most of // the traffic. The gates stop target CREATION only — an existing target's miss was a no-op in // handlePageScheduling anyway — so `discovery_gated` counts gated misses, not denied mints. +// +// The ENTITY gate is the exception to "the gates sit here", and necessarily so: it asks whether +// ANOTHER URL of the same entity has a target, which is a read, and it only means anything for a URL +// with no target of its own. So it runs inside handlePageScheduling, after the existing-row check — +// still detached, still off the response. function maybeSchedule(resource, routeClass, route, botName) { if (routeClass !== PRERENDER || !resource.miss || resource.statusCode !== 200) return; if (route && route.discoverTargets === false) { @@ -453,7 +459,7 @@ function maybeSchedule(resource, routeClass, route, botName) { metrics.discoveryGated('bot', botName); return; } - setImmediate(handlePageScheduling, resource); + setImmediate(handlePageScheduling, resource, route, botName); } // Cache statuses that never looked for a page row, so they can neither prove nor disprove that a @@ -614,7 +620,8 @@ async function renderNow({ url, cacheUrl, deviceType, cacheKey, request, routeSc }; } -async function handlePageScheduling(resource) { +// Exported for tests, which drive the discovery mint end to end against a stubbed Target table. +export async function handlePageScheduling(resource, route, botName) { try { if (isPrerenderCandidate(resource)) { // resource.url is the origin-fetch URL built from the canonical half, so it is @@ -628,6 +635,12 @@ async function handlePageScheduling(resource) { // no row at all is genuinely new. const existingTarget = await Target.get({ id: canonicalUrl, select: 'url' }); if (!existingTarget) { + // THE ENTITY GATE (util/entityGate.js): a URL whose product already has a target in rotation + // under another URL is not minted. Evaluated only here — traffic discovery — and only for a + // URL with no row, so a sitemap or any other creation path never meets it. Inert unless the + // route declares `entityPrefix`; under `ingress.entityGate.dryRun` it counts and mints anyway. + const { mint } = await evaluateEntityGate({ url: canonicalUrl, route, botName }); + if (!mint) return; // No explicit time → Target.put jitters the first render across the interval, // so a crawl that discovers many URLs at once doesn't stampede. // Deliberately NO renderInterval: cadence is resolved at schedule time diff --git a/packages/plugin/src/metrics.js b/packages/plugin/src/metrics.js index 5a9271bf..1ffa47cd 100644 --- a/packages/plugin/src/metrics.js +++ b/packages/plugin/src/metrics.js @@ -480,13 +480,13 @@ export const METRICS = Object.freeze({ emittedBy: 'util/unrouted.js, resources/Sitemap.js, http_handlers/response.js, util/backlogSnapshot.js, ' + 'util/demandLadder.js, util/invalidation.js, util/invalidationReenqueue.js, http_handlers/bot_request.js, ' + - 'util/changeProbe.js', + 'util/changeProbe.js, util/entityGate.js', cadence: 'per report flush (unrouted), per finished sitemap run (sitemap_*), per delivery failure ' + '(serve_error, page_age_negative), per snapshot (config_warnings), per stats interval (demand_*), ' + 'per failed epoch read (invalidation_error), per heal attempt (invalidation_reenqueue), ' + 'per finished probe pass (probe_*, cycle_behind included), per gated cacheable miss (discovery_gated), ' + - 'per raw-document store attempt (raw_cache)', + 'per raw-document store attempt (raw_cache), per entity-gate evaluation (entity_gate)', summary: 'Every low-volume operational signal, under one name so a sweep pays one scan for all of them.', usefulFor: 'unrouted = requests served without prerendering, per path bucket: CDN over-forwarding vs. the ' + @@ -522,7 +522,19 @@ export const METRICS = Object.freeze({ 'discovery_gated = cacheable misses whose target creation the discovery gate refused, split by ' + 'which gate (route flag vs bot allowlist) and by bot. This is gated MISSES, not denied mints — ' + 'a miss on an already-known target counts too — so read it as "traffic on URLs held out of the ' + - 'render rotation", the corpus growth the gate is preventing. ' + + 'render rotation", the corpus growth the gate is preventing. The `entity` gate is the exception ' + + 'in scope, not in meaning: it is evaluated only for a URL with NO target row, so its count is ' + + 'refused mints, and it is emitted only when the gate is ARMED (a dry run records would-gate on ' + + 'entity_gate instead, never here). ' + + 'entity_gate = one emit per evaluation of the entity discovery gate (ingress.entityGate, per-route ' + + 'entityPrefix), split by outcome: gated (a sibling URL of the same entity is in rotation; not minted — ' + + 'also counted as discovery_gated/entity), would-gate (the same verdict under dryRun; minted anyway), ' + + 'suppressed-only (every sibling is suppressed; minted — a re-slug\u2019s new URL), no-siblings (a new ' + + 'entity; minted), no-prefix (the route has an entityPrefix and this URL produced no usable match; ' + + 'minted), error (the sibling read threw; minted). The outcomes sum to the mints the gate looked at. ' + + 'THE DRY-RUN NUMBER is would-gate: the renders (and origin document fetches) arming the gate would ' + + 'save, to read against render/outcome suppressed/canonical-mismatch. A route whose evaluations are ' + + 'nearly all no-prefix has a pattern that does not match its URLs. ' + 'raw_cache = one emit per raw-document store attempt, split by outcome: `stored`, `stored-unshared`, ' + 'or the reason it was refused (not-200, staging, has-cookie, content-type, no-store, no-body, ' + 'oversize, capture-failed, write-failed, vary-device). READ THE REFUSALS, not the successes — a route that is ' + @@ -546,7 +558,7 @@ export const METRICS = Object.freeze({ 'Value semantics per series: unrouted, sitemap_*, the probe_* pass counters and the demand_* decision counters ' + '(promoted/demoted/held/skipped_cold/single_rung/promoted_fast/fast/graded) are per-interval/per-run counts whose `total` is the meaningful ' + 'sum (`count` is flushes/runs); serve_error, page_age_negative, invalidation_error, ' + - 'invalidation_reenqueue, probe_canary_trip, probe_invalidated and discovery_gated are counters; config_warnings is a slow gauge (latest value); ' + + 'invalidation_reenqueue, probe_canary_trip, probe_invalidated, discovery_gated and entity_gate are counters; config_warnings is a slow gauge (latest value); ' + 'demand_fill is a per-worker gauge — never sum it, and READ ITS PEAK, NOT ITS MEAN. It is the ' + 'set-bit fraction of the newest visit-filter slot, which resets to ~0 at every slice rollover ' + 'and climbs until the next one, so it is a sawtooth: averaging over a window reports the middle ' + @@ -597,6 +609,7 @@ export const METRICS = Object.freeze({ 'probe_page_mismatch', 'probe_cycle_behind', 'discovery_gated', + 'entity_gate', ], description: 'unrouted = non-prerendered serve counts (see method/type). sitemap_* = per finished run: ' + @@ -608,7 +621,8 @@ export const METRICS = Object.freeze({ 'guardrail ratio\u2019s two halves, promoted_fast = promotions onto a fast rung, plus the ' + 'fill sizing gauge. ' + 'invalidation_error = failed epoch resolutions. invalidation_reenqueue = heal-attempt outcomes. ' + - 'probe_* = change-probe pass counters (see usefulFor). discovery_gated = gated cacheable misses.', + 'probe_* = change-probe pass counters (see usefulFor). discovery_gated = gated cacheable misses. ' + + 'entity_gate = entity discovery gate evaluations, by outcome.', }, method: { name: 'detail', @@ -625,7 +639,9 @@ export const METRICS = Object.freeze({ '(accepted), not-owner/paused/leased (correctly declined), no-schedule/no-target (nothing to ' + 'accelerate; no-schedule on a live URL is the terminal gap reconcile repairs), unhealable, ' + "not-sooner, throttled, error. discovery_gated: which gate refused ('route' = the matched " + - "route's discoverTargets, 'bot' = ingress.discoveryBots). raw_cache: THE OUTCOME — stored, " + + "route's discoverTargets, 'bot' = ingress.discoveryBots, 'entity' = the route's entityPrefix " + + 'found a sibling URL of the same entity in rotation, armed gate only). entity_gate: the outcome ' + + '(gated, would-gate, suppressed-only, no-siblings, no-prefix, error). raw_cache: THE OUTCOME — stored, ' + 'stored-unshared, or the refusal name; this is the slot the console reads that panel from. ' + 'Other series: null.', }, @@ -634,7 +650,7 @@ export const METRICS = Object.freeze({ description: 'unrouted: first path segment (`/blog/*`), `/` for root (null for the overflow row). ' + 'page_age_negative: the device type. invalidation_reenqueue: the invalidation scope literal ' + - 'that triggered the heal. discovery_gated: the bot name. Other series: null.', + 'that triggered the heal. discovery_gated and entity_gate: the bot name. Other series: null.', }, }, }), @@ -846,6 +862,13 @@ export const metrics = Object.freeze({ discoveryGated: (reason, botName) => server.recordAnalytics(true, 'prerender_ops', 'discovery_gated', reason, botName ?? null), + /** + * One evaluation of the entity discovery gate (util/entityGate.js) and its outcome — a + * prerender_ops series. Exactly one per evaluation, so the outcomes sum to the mints it looked at. + */ + entityGate: (outcome, botName) => + server.recordAnalytics(true, 'prerender_ops', 'entity_gate', outcome, botName ?? null), + /** * One raw-document store attempt and what became of it — a prerender_ops series. * diff --git a/packages/plugin/src/util/entityGate.js b/packages/plugin/src/util/entityGate.js new file mode 100644 index 00000000..d1fe8876 --- /dev/null +++ b/packages/plugin/src/util/entityGate.js @@ -0,0 +1,237 @@ +/** + * The entity gate: traffic discovery does not mint a target for a URL whose ENTITY — the product, + * say — is already in the render rotation under another URL. + * + * THE PROBLEM. `Target` is keyed by URL, so two spellings of one product are two unrelated rows. + * Crawlers keep requesting old or invented slugs of products whose correct URL is already tracked + * (`/product/prd-1/old-spelling.jsp` beside `/product/prd-1/new-spelling.jsp`, or a placeholder + * `/product/prd-1/product.jsp`). Discovery mints each one; it renders; the render finds the + * canonical pointing elsewhere and suppresses it (`canonical-mismatch`); it re-renders on the + * suppression recheck; after `maxStrikes` it is deleted, and the next crawler hit mints it again. + * Measured on one deployment: `canonical-mismatch` was 8.2% of all render outcomes, and in a sample + * of 150 such targets EVERY ONE had another target for the same product id under the correct slug. + * Each costs a render slot and an origin document fetch, and none of them produces a page. + * + * THE MECHANISM IS A PREFIX, NOT AN ID. A route declares `entityPrefix`: a regular expression, + * anchored at the start of the URL PATH, whose match is the part of the URL that identifies one + * entity (`^/product/prd-[^/]+/` on `/product/prd-`). The lookup prefix is the URL's origin plus + * that match, and every Target row whose key starts with it is a SIBLING. Because `Target` is keyed + * by the canonical URL, "rows sharing a prefix" is one bounded primary-key range read — no second + * table, no index, nothing to keep in step with the registry (issue #166 discusses the id-keyed + * table this deliberately is not). + * + * THE MATCH MUST END ON `/`, and this is enforced at runtime, not merely advised. A prefix that + * stops mid-segment is a prefix of OTHER entities' URLs: `…/prd-123` is a string prefix of + * `…/prd-1234/any.jsp`, so product 1234's target would gate product 123 — permanently, since + * nothing about 1234 ever changes that. A match that does not end in `/` is therefore treated as + * no match at all (outcome `no-prefix`, minted exactly as before), and compiling a pattern whose + * source does not end in `/` warns. The warning is a heads-up rather than a rejection because a + * pattern like `^/product/prd-\d+/?` is usable — its match ends in `/` whenever the URL continues + * past the id — and the runtime rule already refuses the matches that do not. For the same reason a + * match that does not extend past a `prefix` route's own path is refused: it names the whole route, + * not one entity. + * + * THE DECISION. A sibling IN ROTATION (`state` not `suppressed`) means the entity already has a + * target that will render, so this URL is not minted. A SUPPRESSED sibling does not gate, and that + * is what keeps the gate from ever blocking an entity's correct URL for good: + * + * - A genuine re-slug. The old slug's target is still active until its next render proves the + * canonical moved and suppresses it; until then the new slug is gated. Once it is suppressed, + * the next crawler hit on the new slug mints normally. The cost is a BOUNDED DELAY — one render + * cycle of the old target (up to its cadence; a freshly minted target's first render is jittered + * across a whole interval) — during which the new URL is served exactly as an unknown URL always + * has been: a miss proxies the origin. + * - A sitemap that lists a non-canonical URL. Its target is listed but suppressed, and the page it + * canonicalizes to must still be discoverable. Treating "listed" as gating on its own would block + * that correct URL forever — the listing re-creates the suppressed row on every refresh — which is + * why `sitemapUrl` plays no part in the decision. A listed target in rotation gates because it is + * in rotation, not because it is listed. + * + * DISCOVERY ONLY. Sitemap ingestion, redirect adoption, the REST API and suppression all create + * targets without passing through here — the declared corpus is always created. The single call + * site is `handlePageScheduling` in http_handlers/bot_request.js, which a test pins. + * + * THE READ IS OFF THE SERVE PATH AND CANNOT HANG. It runs inside the detached discovery step + * (`setImmediate` after the response resolved), only for a URL that has NO target row at all, only + * on a route that opted in. `Target` is not residency-pinned — only `RenderSchedule` is — so the + * read is node-local; `replicateFrom: false` rides along anyway, as the second argument (Harper's + * search path ignores unknown query fields, so inside the query it would look honoured and do + * nothing), so that a future residency pin on `Target` cannot turn this into an untimed cross-node + * fetch. The range is ONE-SIDED (`>= prefix`) with the upper bound enforced while consuming — + * rows are key-ordered, so the first key outside the prefix ends the read — because a two-sided + * primary-key range collapses to a filtered intersection rather than a seek (see + * `PrerenderAdmin.listPagesInner`). It is `limit`-bounded, and the loop body never awaits, so the + * cursor is closed before anything else happens (util/scan.js explains why that matters). + * + * EVERY FAILURE MINTS, i.e. behaves exactly as discovery did before this existed. A read that + * throws, an unreadable row, a URL the pattern does not match: all fall through to the mint. The gate + * can only ever SAVE work; it has no failure mode that takes a URL out of the rotation it would have + * joined without it, other than the bounded re-slug delay above. + */ + +import { config, getLogger } from '../config.js'; +import { metrics } from '../metrics.js'; + +/** The delimiter an entity prefix must end on — see the module comment for why it is enforced. */ +export const ENTITY_DELIMITER = '/'; + +/** + * Every outcome of one evaluation, as recorded on `prerender_ops` / `entity_gate`. Exactly one per + * evaluation, so the series sums to "discovery mints the gate looked at". + */ +export const EntityGateOutcome = Object.freeze({ + /** A sibling is in rotation and the gate is armed: not minted. Also counted as `discovery_gated`/`entity`. */ + GATED: 'gated', + /** A sibling is in rotation but `dryRun` is on: minted anyway, and counted — the rollout number. */ + WOULD_GATE: 'would-gate', + /** Siblings exist but every one is suppressed: minted (a re-slug's new URL, or a crawler's). */ + SUPPRESSED_ONLY: 'suppressed-only', + /** No sibling at all: a genuinely new entity, minted. */ + NO_SIBLINGS: 'no-siblings', + /** The route has an `entityPrefix` but this URL produced no usable prefix: minted. */ + NO_PREFIX: 'no-prefix', + /** The sibling read threw: minted (fail open), and logged. */ + ERROR: 'error', +}); + +// The only fields the decision reads. `url` is the key (a row without a readable one is skipped); +// `state` is the verdict. An ARRAY, because a string `select` projects to the bare scalar. +export const SIBLING_SELECT = Object.freeze(['url', 'state']); + +/** + * Compile a route's `entityPrefix` source. STICKY (`y`), which anchors the match at the start of + * the path whether or not the author wrote `^` — "anchored at the path root" is the contract, and a + * pattern that could match mid-path would name an entity by something other than its prefix. + * Throws on an invalid pattern; the route compiler catches it and drops the field. + */ +export const compileEntityPrefix = (source) => new RegExp(source, 'y'); + +/** + * Whether a pattern's SOURCE ends on the delimiter. A compile-time heuristic only: the runtime rule + * in `entityPrefixOf` checks the actual match, which is what makes it safe; this exists so an author + * learns at config time that some (or all) of their matches will be refused. + */ +export const endsOnDelimiter = (source) => typeof source === 'string' && source.endsWith(ENTITY_DELIMITER); + +/** + * The sibling-lookup prefix for a canonical URL on a route, or `null` when there is none to use. + * + * `url` is a Target primary key (a canonical URL-half — see util/url.js), so its origin plus the + * matched path text is a string prefix of every canonical URL under the same entity. The final + * `startsWith` is a guard, not a formality: it proves the prefix really is a prefix of the key the + * siblings are compared against, so a parse that disagreed with the canonical bytes could only ever + * make the gate do nothing. + */ +export const entityPrefixOf = (url, route) => { + const pattern = route?.entityPrefix; + if (!(pattern instanceof RegExp) || typeof url !== 'string') return null; + const parsed = URL.parse(url); + if (!parsed) return null; + pattern.lastIndex = 0; + const match = pattern.exec(parsed.pathname); + pattern.lastIndex = 0; + const text = match?.[0]; + if (!text || !text.endsWith(ENTITY_DELIMITER)) return null; + // A match no longer than a prefix route's own path covers the whole route, not one entity. + if (route.match === 'prefix' && typeof route.path === 'string' && text.length <= route.path.length) return null; + const prefix = parsed.origin + text; + return url.startsWith(prefix) ? prefix : null; +}; + +/** A sibling in rotation keeps its entity rendering, so it gates. A suppressed one never does. */ +export const inRotation = (row) => row?.state !== 'suppressed'; + +/** + * How many rows one evaluation reads. FIXED, not configuration, because the answer was measured and a + * knob for it would only be a way to get it wrong: across 150 suppressed old-slug targets on a + * production catalog, the first in-rotation sibling under the entity prefix sat at key-order position + * 1 for 135, position 2 for 14 and position 3 for 1 — so 3 rows found it for 150/150 (1 row: 90%, + * 2 rows: 99%), and no entity had more than 4 siblings at all. Past the limit the gate MINTS (the + * pre-gate behaviour), never refuses on a guess, so a miss costs one render, not a lost URL. + */ +export const SIBLING_READ_LIMIT = 3; + +/** + * Look at `url`'s siblings under `prefix`: `{ blocker, seen }`, where `blocker` is the key of the first + * sibling in rotation (null when none was found) and `seen` how many siblings were read. Stops at the + * first sibling in rotation — one is enough to decide — at the first key outside the prefix, or after + * SIBLING_READ_LIMIT rows. `url` itself is never its own sibling (it has no row when discovery gets + * here, but a concurrent mint may have just written one; it then costs a row, which only shortens the + * look — the fail-open direction). + * + * THE LOOP BODY DOES NOT AWAIT. The cursor is released (by the `break`, or by exhaustion) before the + * caller does anything else — the scan-then-write rule from util/scan.js. No rows are kept: the decision + * needs only the first blocker's key and whether any sibling existed. + */ +export const readSiblings = async ({ table, prefix, url }) => { + let seen = 0; + for await (const row of table.search( + { + conditions: [{ attribute: 'url', comparator: 'greater_than_equal', value: prefix }], + sort: { attribute: 'url' }, + select: [...SIBLING_SELECT], + limit: SIBLING_READ_LIMIT, + }, + { replicateFrom: false } + )) { + const key = row?.url; + // An unreadable key can be neither compared nor trusted; skip it (the read stays bounded by the + // limit). If the store instead ENDS the iterator at such a row, fewer siblings are seen and the + // gate mints — the fail-open direction. + if (typeof key !== 'string') continue; + // BREAK, never `continue`: ascending order means one key outside the prefix proves every later + // key is outside it too. + if (!key.startsWith(prefix)) break; + if (key === url) continue; + seen++; + if (inRotation(row)) return { blocker: key, seen }; + } + return { blocker: null, seen }; +}; + +/** + * Evaluate the gate for a discovery mint of `url` on `route`. Returns `{ mint, outcome, prefix, + * blocker }`: `mint` is the only field the caller must act on; `outcome` is null when the gate did + * not evaluate at all (switched off, or the route has no `entityPrefix`), which records nothing. + * + * All I/O is injectable (`table`, `gate`), so every decision is testable without a live Harper. + */ +export async function evaluateEntityGate({ + url, + route, + botName = null, + table = globalThis.databases?.render_service?.Target, + gate = config.ingress.entityGate, +}) { + if (!gate?.enabled || !(route?.entityPrefix instanceof RegExp)) { + return { mint: true, outcome: null, prefix: null, blocker: null }; + } + + const decided = (outcome, mint, prefix = null, blocker = null) => { + metrics.entityGate(outcome, botName); + return { mint, outcome, prefix, blocker }; + }; + + const prefix = entityPrefixOf(url, route); + if (!prefix) return decided(EntityGateOutcome.NO_PREFIX, true); + + let blocker; + let seen; + try { + ({ blocker, seen } = await readSiblings({ table, prefix, url })); + } catch (e) { + getLogger().warn?.( + `[prerender] entity gate: sibling read under ${prefix} failed (${e?.message ?? String(e)}); minting ${url} as before` + ); + return decided(EntityGateOutcome.ERROR, true, prefix); + } + + if (!seen) return decided(EntityGateOutcome.NO_SIBLINGS, true, prefix); + if (!blocker) return decided(EntityGateOutcome.SUPPRESSED_ONLY, true, prefix); + if (gate.dryRun) return decided(EntityGateOutcome.WOULD_GATE, true, prefix, blocker); + + // Armed and refused. Counted on `discovery_gated` too, beside the route and bot gates, so every + // view of "what the gates held out of the rotation" includes it without learning a new series. + metrics.discoveryGated('entity', botName); + return decided(EntityGateOutcome.GATED, false, prefix, blocker); +} diff --git a/packages/plugin/src/util/routeClass.js b/packages/plugin/src/util/routeClass.js index d207be85..1e3bd6db 100644 --- a/packages/plugin/src/util/routeClass.js +++ b/packages/plugin/src/util/routeClass.js @@ -33,6 +33,7 @@ */ import { config, getLogger } from '../config.js'; +import { compileEntityPrefix, endsOnDelimiter } from './entityGate.js'; export const PRERENDER = 'prerender'; export const PASSTHROUGH = 'passthrough'; @@ -241,6 +242,50 @@ const compileEntry = (raw, source, warn) => { } } + // Optional per-route ENTITY PREFIX — what the discovery gate treats as "the same product" (see + // util/entityGate.js). Same drop-the-FIELD rule as the fields above, and dropping it is exactly the + // pre-gate behaviour: every unknown URL on the route is minted. Compiled here, once per config + // apply, so the discovery path never parses a pattern. + let entityPrefix = null; + if (raw.entityPrefix !== undefined && raw.entityPrefix !== null) { + if (mode === PASSTHROUGH) { + warn( + `ignoring entityPrefix on passthrough route "${raw.match} ${raw.path}" — a passthrough route is never ` + + `scheduled, so it never discovers targets and there is nothing to gate` + ); + } else if (typeof raw.entityPrefix !== 'string' || raw.entityPrefix === '') { + warn( + `ignoring entityPrefix on route "${raw.match} ${raw.path}" — expected a non-empty regular expression ` + + `string, got ${String(raw.entityPrefix)}` + ); + } else { + try { + entityPrefix = compileEntityPrefix(raw.entityPrefix); + } catch (e) { + warn( + `ignoring entityPrefix on route "${raw.match} ${raw.path}" — not a valid regular expression ` + + `(${e?.message ?? String(e)})` + ); + } + // Warned, not dropped: the RUNTIME rule is what makes this safe — a match that does not end in + // "/" is refused per URL — so a pattern like `…\d+/?` still works wherever the URL continues past + // the id. The warning is so the author learns that some (or all) of their matches will be refused. + if (entityPrefix && !endsOnDelimiter(raw.entityPrefix)) { + warn( + `entityPrefix "${raw.entityPrefix}" on route "${raw.match} ${raw.path}" does not end in "/". A match ` + + `must end on "/" or it would be a string prefix of OTHER entities' URLs (prd-123 of prd-1234), so ` + + `any match that does not is ignored and that URL is discovered as if no entityPrefix were set` + ); + } + if (entityPrefix && discoverTargets === false) { + warn( + `entityPrefix on route "${raw.match} ${raw.path}" does nothing: the route sets discoverTargets: false, ` + + `so no URL on it is ever discovered for the entity gate to refuse` + ); + } + } + } + return { match: raw.match, path: raw.path, @@ -252,6 +297,7 @@ const compileEntry = (raw, source, warn) => { departureAction, arrivalAction, rawCache, + entityPrefix, source, }; }; diff --git a/packages/plugin/test/entityGate.test.js b/packages/plugin/test/entityGate.test.js new file mode 100644 index 00000000..dd77c38a --- /dev/null +++ b/packages/plugin/test/entityGate.test.js @@ -0,0 +1,519 @@ +import { test, before, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** + * The entity discovery gate (util/entityGate.js) and the per-route `entityPrefix` it reads. + * + * What is pinned here: + * - the route field is validated like every other optional route field: a bad value drops the + * FIELD with a warning, never the route, and a passthrough route refuses it; + * - a prefix must end on "/" at RUNTIME, so product 123 can never be gated by product 1234; + * - the decision: a sibling in rotation gates (listed or not), suppressed-only siblings mint, no + * siblings mint, the URL is never its own sibling, a dry run mints and counts; + * - the read is one bounded, one-sided, node-local primary-key range with the minimal projection, + * and its cursor is closed before anything is recorded; + * - discovery is the ONLY path that consults it — sitemap-driven creation never does. + */ + +const analytics = []; +let openCursors = 0; +let cursorOpenAtEmit = false; + +globalThis.server = { + hostname: 'test-node', + nodes: [], + config: { http: { port: 9926 } }, + recordAnalytics: (...args) => { + if (openCursors > 0) cursorOpenAtEmit = true; + analytics.push(args); + }, +}; +globalThis.logger = { debug() {}, info() {}, warn() {}, error() {}, trace() {} }; +globalThis.Resource = class {}; + +// The stubbed `Target` base the resource class extends. `get`/`put` are the calls +// `handlePageScheduling` makes; `search` is the sibling read. Every row lives in `rows`. +class TargetBase { + static rows = new Map(); + static puts = []; + static searches = []; + static async get({ id }) { + return TargetBase.rows.has(id) ? { url: id } : null; + } + static async put(id) { + TargetBase.puts.push(id); + } + static search(query, context) { + return fakeSearch([...TargetBase.rows.values()], query, context, TargetBase.searches); + } +} + +globalThis.databases = { + coordination: { + SharedBuffer: { + primaryStore: { getUserSharedBuffer: (_key, buf) => buf, tryLock: () => true, unlock() {} }, + }, + }, + render_service: { Target: TargetBase, QueueControl: class {} }, + render_schedule: { RenderSchedule: class {} }, + page_cache: { PrerenderedPage: class {} }, + probe_state: { ProbeState: class {}, RenderExpectation: class {} }, +}; + +/** + * An in-memory stand-in for `Table.search` over a string primary key: ascending key order, the one + * condition the gate issues, the projection, the limit — and cursor accounting, so a test can prove + * the cursor is released (by `return()` on an early break, or by exhaustion). + */ +function fakeSearch(allRows, query, context, log) { + log.push({ query, context }); + const [condition] = query.conditions; + assert.equal(query.conditions.length, 1, 'the sibling read must be ONE-SIDED — a two-sided PK range is a scan'); + const rows = allRows + .filter((row) => typeof row.url !== 'string' || row.url >= condition.value) + .sort((a, b) => String(a.url ?? a.sortAs).localeCompare(String(b.url ?? b.sortAs))) + .slice(0, query.limit); + let index = 0; + let open = true; + openCursors++; + const close = () => { + if (open) { + open = false; + openCursors--; + } + }; + return { + [Symbol.asyncIterator]() { + return this; + }, + async next() { + if (index >= rows.length) { + close(); + return { done: true, value: undefined }; + } + const row = rows[index++]; + return { done: false, value: Object.fromEntries(query.select.map((key) => [key, row[key]])) }; + }, + async return() { + close(); + return { done: true, value: undefined }; + }, + }; +} + +/** A standalone table over `rows` — for the unit tests that do not go through the Target stub. */ +const tableOf = (rows) => { + const searches = []; + return { searches, search: (query, context) => fakeSearch(rows, query, context, searches) }; +}; + +const ORIGIN = 'https://www.example.com'; +const U = `${ORIGIN}/product/prd-123/new-spelling.jsp`; +const PATTERN = '^/product/prd-[^/]+/'; + +let applyOptions; +let config; +let matchRoute; +let inspectRoutes; +let evaluateEntityGate; +let entityPrefixOf; +let EntityGateOutcome; +let SIBLING_SELECT; +let SIBLING_READ_LIMIT; +let handlePageScheduling; + +before(async () => { + ({ applyOptions, config } = await import('../src/config.js')); + ({ matchRoute, inspectRoutes } = await import('../src/util/routeClass.js')); + ({ evaluateEntityGate, entityPrefixOf, EntityGateOutcome, SIBLING_SELECT, SIBLING_READ_LIMIT } = await import( + '../src/util/entityGate.js' + )); + ({ handlePageScheduling } = await import('../src/http_handlers/bot_request.js')); +}); + +const productRoute = (extra = {}) => ({ match: 'prefix', path: '/product/prd-', queryParams: [], ...extra }); + +// `entityPrefix: undefined` must mean ABSENT, so presence is tested with `in` rather than left to a +// destructuring default (which would substitute PATTERN for an explicit undefined). +const configure = (options = {}) => { + const { gate = {}, route = {} } = options; + const entityPrefix = 'entityPrefix' in options ? options.entityPrefix : PATTERN; + return applyOptions({ + ingress: { + mode: 'forwarded', + deviceTypeSource: 'path', + excludePathPatterns: [], + routes: [productRoute({ ...(entityPrefix === undefined ? {} : { entityPrefix }), ...route })], + entityGate: { dryRun: false, ...gate }, + }, + }); +}; + +const route = () => matchRoute('/product/prd-123/x.jsp'); + +beforeEach(() => { + analytics.length = 0; + cursorOpenAtEmit = false; + TargetBase.rows = new Map(); + TargetBase.puts = []; + TargetBase.searches = []; + configure(); +}); + +const put = (url, fields = {}) => TargetBase.rows.set(url, { url, sitemapUrl: null, state: null, ...fields }); +const ops = (series) => analytics.filter(([, metric, path]) => metric === 'prerender_ops' && path === series); +const outcomes = () => ops('entity_gate').map(([, , , outcome]) => outcome); + +// ── the route field ──────────────────────────────────────────────────────────────────────────── + +test('a valid entityPrefix compiles onto the route, anchored at the path root', () => { + const entry = route(); + assert.ok(entry.entityPrefix instanceof RegExp); + assert.ok(entry.entityPrefix.sticky, 'sticky, so a match can only start at the path root'); + assert.deepEqual(inspectRoutes([productRoute({ entityPrefix: PATTERN })], []).warnings, []); +}); + +test('an absent or null entityPrefix is simply off — no field, no warning', () => { + for (const entityPrefix of [undefined, null]) { + const raw = productRoute(entityPrefix === undefined ? {} : { entityPrefix }); + assert.deepEqual(inspectRoutes([raw], []).warnings, []); + configure({ entityPrefix }); + assert.equal(route().entityPrefix, null); + } +}); + +test('an invalid regular expression drops the FIELD with a warning, never the route', () => { + const { warnings, prerender, dropped } = inspectRoutes([productRoute({ entityPrefix: '^/product/prd-[/' })], []); + assert.equal(prerender, 1); + assert.equal(dropped, 0); + assert.ok(warnings.some((w) => w.includes('entityPrefix') && w.includes('not a valid regular expression'))); + configure({ entityPrefix: '^/product/prd-[/' }); + assert.equal(route().mode, 'prerender', 'the route still prerenders'); + assert.equal(route().entityPrefix, null); +}); + +test('a non-string or empty entityPrefix drops the field with a warning', () => { + for (const bad of [123, true, '', ['^/a/'], { source: '^/a/' }]) { + const { warnings, prerender } = inspectRoutes([productRoute({ entityPrefix: bad })], []); + assert.equal(prerender, 1); + assert.ok( + warnings.some((w) => w.includes('entityPrefix') && w.includes('expected a non-empty regular expression')), + `expected a type warning for ${JSON.stringify(bad)}, got ${JSON.stringify(warnings)}` + ); + } +}); + +test('a passthrough route refuses entityPrefix — it never discovers anything', () => { + const raw = { match: 'prefix', path: '/help/', mode: 'passthrough', entityPrefix: '^/help/[^/]+/' }; + const { warnings } = inspectRoutes([raw], []); + assert.ok(warnings.some((w) => w.includes('ignoring entityPrefix on passthrough route'))); + applyOptions({ ingress: { mode: 'forwarded', excludePathPatterns: [], routes: [raw] } }); + assert.equal(matchRoute('/help/a/b').entityPrefix, null); +}); + +test('a pattern that does not end in "/" is KEPT but warned about — the runtime rule is what refuses it', () => { + const { warnings } = inspectRoutes([productRoute({ entityPrefix: '^/product/prd-\\d+' })], []); + assert.ok(warnings.some((w) => w.includes('does not end in "/"') && w.includes('prd-1234'))); + configure({ entityPrefix: '^/product/prd-\\d+' }); + assert.ok(route().entityPrefix instanceof RegExp); +}); + +test('entityPrefix on a route with discoverTargets: false is warned about as a no-op', () => { + const { warnings } = inspectRoutes([productRoute({ entityPrefix: PATTERN, discoverTargets: false })], []); + assert.ok(warnings.some((w) => w.includes('entityPrefix') && w.includes('does nothing'))); +}); + +// ── the prefix ───────────────────────────────────────────────────────────────────────────────── + +test('the lookup prefix is the URL origin plus the matched path text', () => { + assert.equal(entityPrefixOf(U, route()), `${ORIGIN}/product/prd-123/`); + // The port is part of the origin, and of the canonical key. + assert.equal( + entityPrefixOf('https://www.example.com:8443/product/prd-9/x.jsp', route()), + 'https://www.example.com:8443/product/prd-9/' + ); +}); + +test('the match is anchored at the path root even without a leading ^', () => { + configure({ entityPrefix: '/product/prd-[^/]+/' }); + assert.equal(entityPrefixOf(U, route()), `${ORIGIN}/product/prd-123/`); + // A pattern that could only match MID-path matches nothing. + configure({ entityPrefix: 'prd-[^/]+/' }); + assert.equal(entityPrefixOf(U, route()), null); +}); + +test('PREFIX COLLISION: product 123 is never a sibling of product 1234, whatever the pattern', () => { + // A well-formed pattern: the prefix ends on "/", so it is not a string prefix of 1234's URL. + const prefix = entityPrefixOf(U, route()); + assert.ok(!`${ORIGIN}/product/prd-1234/other.jsp`.startsWith(prefix)); + + // An undelimited pattern WOULD produce `…/prd-123`, a string prefix of every prd-1234 URL. The + // runtime rule refuses that match outright. + configure({ entityPrefix: '^/product/prd-\\d+' }); + assert.equal(entityPrefixOf(U, route()), null); +}); + +test('an optional trailing "/" is usable: refused on the bare id URL, used where the URL continues', () => { + configure({ entityPrefix: '^/product/prd-\\d+/?' }); + assert.equal(entityPrefixOf(`${ORIGIN}/product/prd-123`, route()), null); + assert.equal(entityPrefixOf(U, route()), `${ORIGIN}/product/prd-123/`); +}); + +test('a match that does not extend past a prefix route’s own path names the whole route and is refused', () => { + applyOptions({ + ingress: { + mode: 'forwarded', + excludePathPatterns: [], + routes: [{ match: 'prefix', path: '/product/', queryParams: [], entityPrefix: '^/product/' }], + }, + }); + assert.equal(entityPrefixOf(`${ORIGIN}/product/prd-1/x.jsp`, matchRoute('/product/prd-1/x.jsp')), null); +}); + +// ── the decision ─────────────────────────────────────────────────────────────────────────────── + +const gate = (overrides = {}) => ({ enabled: true, dryRun: false, ...overrides }); + +test('a sitemap-listed sibling in rotation GATES: not minted, counted on both series', async () => { + const table = tableOf([ + { url: `${ORIGIN}/product/prd-123/old-spelling.jsp`, sitemapUrl: `${ORIGIN}/sitemap-1.xml`, state: null }, + ]); + const result = await evaluateEntityGate({ url: U, route: route(), botName: 'Googlebot', table, gate: gate() }); + assert.equal(result.mint, false); + assert.equal(result.outcome, EntityGateOutcome.GATED); + assert.equal(result.blocker, `${ORIGIN}/product/prd-123/old-spelling.jsp`); + assert.deepEqual(outcomes(), ['gated']); + assert.deepEqual(ops('discovery_gated'), [[true, 'prerender_ops', 'discovery_gated', 'entity', 'Googlebot']]); +}); + +test('an active UNLISTED sibling gates too', async () => { + const table = tableOf([{ url: `${ORIGIN}/product/prd-123/new.jsp`, sitemapUrl: null, state: null }]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.mint, false); + assert.equal(result.outcome, 'gated'); +}); + +test('SUPPRESSED-ONLY siblings mint — a re-slug is delayed, never blocked for good', async () => { + const table = tableOf([ + { url: `${ORIGIN}/product/prd-123/a.jsp`, state: 'suppressed' }, + // Listed AND suppressed: a sitemap listing a non-canonical URL must not block the canonical one. + { url: `${ORIGIN}/product/prd-123/b.jsp`, sitemapUrl: `${ORIGIN}/sitemap-1.xml`, state: 'suppressed' }, + ]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'suppressed-only'); + assert.deepEqual(ops('discovery_gated'), []); +}); + +test('no siblings mints — including when the NEIGHBOURING product (prd-1234) is tracked', async () => { + const table = tableOf([ + { url: `${ORIGIN}/product/prd-1234/other.jsp`, state: null }, + { url: `${ORIGIN}/product/prd-12/other.jsp`, state: null }, + ]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'no-siblings'); +}); + +test('the URL is never its own sibling', async () => { + const table = tableOf([{ url: U, state: null }]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'no-siblings'); +}); + +test('DRY RUN: the same verdict mints anyway and counts would-gate — never discovery_gated', async () => { + const table = tableOf([{ url: `${ORIGIN}/product/prd-123/new.jsp`, state: null }]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate({ dryRun: true }) }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'would-gate'); + assert.deepEqual(outcomes(), ['would-gate']); + assert.deepEqual(ops('discovery_gated'), []); +}); + +test('a route without entityPrefix is untouched: no read, nothing recorded', async () => { + configure({ entityPrefix: undefined }); + const table = tableOf([{ url: `${ORIGIN}/product/prd-123/new.jsp`, state: null }]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.deepEqual(result, { mint: true, outcome: null, prefix: null, blocker: null }); + assert.equal(table.searches.length, 0); + assert.deepEqual(analytics, []); + // And with no matched route at all (prefix ingress with no routes declared). + assert.equal((await evaluateEntityGate({ url: U, route: null, table, gate: gate() })).outcome, null); +}); + +test('the master switch off: no read, nothing recorded, minted', async () => { + const table = tableOf([{ url: `${ORIGIN}/product/prd-123/new.jsp`, state: null }]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate({ enabled: false }) }); + assert.equal(result.mint, true); + assert.equal(table.searches.length, 0); + assert.deepEqual(analytics, []); +}); + +test('a URL the pattern does not match mints and is counted no-prefix', async () => { + configure({ entityPrefix: '^/product/prd-\\d+/' }); + const table = tableOf([]); + const result = await evaluateEntityGate({ + url: `${ORIGIN}/product/prd-abc/x.jsp`, + route: matchRoute('/product/prd-abc/x.jsp'), + table, + gate: gate(), + }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'no-prefix'); + assert.equal(table.searches.length, 0); +}); + +test('a read that throws FAILS OPEN: minted, counted as error', async () => { + const table = { + search() { + throw new Error('store unavailable'); + }, + }; + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.mint, true); + assert.equal(result.outcome, 'error'); +}); + +test('an unreadable row is skipped, never taken for a sibling or a cursor', async () => { + const table = tableOf([ + { url: undefined, sortAs: `${ORIGIN}/product/prd-123/0`, state: null }, + { url: `${ORIGIN}/product/prd-123/a.jsp`, state: 'suppressed' }, + ]); + const result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.outcome, 'suppressed-only'); +}); + +// ── the read ─────────────────────────────────────────────────────────────────────────────────── + +test('the read is one bounded, one-sided, node-local PK range with the minimal projection', async () => { + const table = tableOf([{ url: `${ORIGIN}/product/prd-123/a.jsp`, state: 'suppressed' }]); + await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(table.searches.length, 1); + const [{ query, context }] = table.searches; + assert.deepEqual(query.conditions, [ + { attribute: 'url', comparator: 'greater_than_equal', value: `${ORIGIN}/product/prd-123/` }, + ]); + assert.deepEqual(query.sort, { attribute: 'url' }); + assert.deepEqual(query.select, [...SIBLING_SELECT]); + assert.deepEqual(query.select, ['url', 'state']); + assert.equal(SIBLING_READ_LIMIT, 3, 'measured: the first sibling in rotation was within 3 keys for 150/150 entities'); + assert.equal(query.limit, SIBLING_READ_LIMIT, 'a fixed 3-row read — no knob'); + // SECOND argument: Harper ignores unknown query fields, so inside the query it would do nothing. + assert.deepEqual(context, { replicateFrom: false }); + assert.equal('replicateFrom' in query, false); +}); + +test('the cursor is released before anything is recorded — on an early break and on exhaustion', async () => { + // Early break: the first sibling in rotation decides. + let table = tableOf([ + { url: `${ORIGIN}/product/prd-123/a.jsp`, state: null }, + { url: `${ORIGIN}/product/prd-123/b.jsp`, state: null }, + ]); + await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(openCursors, 0); + assert.equal(cursorOpenAtEmit, false); + + // Exhaustion. + table = tableOf([]); + await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(openCursors, 0); + assert.equal(cursorOpenAtEmit, false); +}); + +test('the read stops at the first sibling in rotation, and the 3-row limit bounds the suppressed case', async () => { + const dead = (n) => + Array.from({ length: n }, (_, i) => ({ url: `${ORIGIN}/product/prd-123/dead-${i}.jsp`, state: 'suppressed' })); + const live = { url: `${ORIGIN}/product/prd-123/zz-live.jsp`, state: null }; + // Two dead slugs ahead of the live one: the live one is the third row read, so the gate finds it. + let result = await evaluateEntityGate({ url: U, route: route(), table: tableOf([...dead(2), live]), gate: gate() }); + assert.equal(result.outcome, 'gated'); + assert.equal(result.blocker, live.url); + // Three dead slugs ahead of it: the live one is past the read, and the gate MINTS (the pre-gate + // behaviour) — it never refuses on a guess. + result = await evaluateEntityGate({ url: U, route: route(), table: tableOf([...dead(3), live]), gate: gate() }); + assert.equal(result.outcome, 'suppressed-only'); + assert.equal(result.mint, true); + // The first sibling in rotation ends the read: nothing after it is consumed. + const table = tableOf([live, ...dead(2)]); + result = await evaluateEntityGate({ url: U, route: route(), table, gate: gate() }); + assert.equal(result.outcome, 'gated'); +}); + +// ── the discovery path ───────────────────────────────────────────────────────────────────────── + +const originMiss = (url) => ({ + miss: true, + statusCode: 200, + url, + headers: { 'content-type': 'text/html; charset=utf-8' }, +}); + +test('discovery, ARMED: a URL whose product has a target in rotation is not minted', async () => { + put(`${ORIGIN}/product/prd-123/old-spelling.jsp`); + await handlePageScheduling(originMiss(U), route(), 'Googlebot'); + assert.deepEqual(TargetBase.puts, []); + assert.deepEqual(outcomes(), ['gated']); +}); + +test('discovery, DRY RUN (the default): minted exactly as before, and counted', async () => { + configure({ gate: { dryRun: true } }); + put(`${ORIGIN}/product/prd-123/old-spelling.jsp`); + await handlePageScheduling(originMiss(U), route(), 'Googlebot'); + assert.deepEqual(TargetBase.puts, [U]); + assert.deepEqual(outcomes(), ['would-gate']); +}); + +test('discovery on a route without entityPrefix: minted, no sibling read, nothing recorded', async () => { + configure({ entityPrefix: undefined }); + put(`${ORIGIN}/product/prd-123/old-spelling.jsp`); + await handlePageScheduling(originMiss(U), route(), 'Googlebot'); + assert.deepEqual(TargetBase.puts, [U]); + assert.equal(TargetBase.searches.length, 0); + assert.deepEqual(outcomes(), []); +}); + +test('discovery of a URL that already has a row: no sibling read at all', async () => { + put(U, { state: 'suppressed' }); + put(`${ORIGIN}/product/prd-123/old-spelling.jsp`); + await handlePageScheduling(originMiss(U), route(), 'Googlebot'); + assert.deepEqual(TargetBase.puts, []); + assert.equal(TargetBase.searches.length, 0); +}); + +test('discovery with suppressed-only siblings mints the new URL', async () => { + put(`${ORIGIN}/product/prd-123/old-spelling.jsp`, { state: 'suppressed' }); + await handlePageScheduling(originMiss(U), route(), 'Googlebot'); + assert.deepEqual(TargetBase.puts, [U]); + assert.deepEqual(outcomes(), ['suppressed-only']); +}); + +// ── scope: discovery only ────────────────────────────────────────────────────────────────────── + +test('discovery is the ONLY caller — sitemap-driven (and every other) creation never meets the gate', async () => { + const SRC = new URL('../src/', import.meta.url).pathname; + const files = (await readdir(SRC, { recursive: true })).filter((f) => f.endsWith('.js')); + const callers = []; + for (const file of files) { + const source = await readFile(join(SRC, file), 'utf8'); + if (file.endsWith('entityGate.js')) continue; + if (/\bevaluateEntityGate\b/.test(source)) callers.push(file); + } + assert.deepEqual(callers, [join('http_handlers', 'bot_request.js')]); + + const botRequest = await readFile(join(SRC, 'http_handlers', 'bot_request.js'), 'utf8'); + const calls = [...botRequest.matchAll(/await evaluateEntityGate\(/g)]; + assert.equal(calls.length, 1, 'exactly one call site'); + // ...and it sits inside handlePageScheduling, after the existing-row check. + const body = botRequest.slice(botRequest.indexOf('export async function handlePageScheduling')); + assert.ok(body.indexOf('await evaluateEntityGate(') > body.indexOf('if (!existingTarget)')); +}); + +// ── config ───────────────────────────────────────────────────────────────────────────────────── + +test('ingress.entityGate defaults: on, DRY RUN — and the read size is not configuration', () => { + applyOptions({}); + assert.deepEqual(config.ingress.entityGate, { enabled: true, dryRun: true }); +});