From c590635f3dd4dd9ad031cba609435bbc4f4a33d4 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 24 Aug 2026 18:57:05 -0400 Subject: [PATCH] =?UTF-8?q?feat(plugin):=20per-route=20demandFloor=20?= =?UTF-8?q?=E2=80=94=20bound=20the=20demand=20ladder=20per=20route;=20v0.5?= =?UTF-8?q?5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A breadth-sweeping crawler that recrawls everything daily makes 'visited in consecutive windows' true corpus-wide, so global ladder rungs grant the fast cadence at corpus scale to whichever route that crawler sweeps. demandFloor makes the fastest reachable rung per-route policy: rungs faster than a route's floor are unreachable for its pages, so fast global rungs can serve a small churning corpus (listing pages) without dragging a large swept one (product pages) onto them. Consulted per ladder decision (live, like renderInterval); a stored rung below a newly-raised floor reads as the floor immediately (resolveEffectiveInterval) and re-stamps on its next decision (rungIndexOf snaps into the floored list). Co-Authored-By: Claude Fable 5 --- package-lock.json | 2 +- packages/plugin/package.json | 2 +- packages/plugin/src/configSchema.js | 15 ++++++- packages/plugin/src/util/demandLadder.js | 24 ++++++++--- packages/plugin/src/util/routeClass.js | 50 ++++++++++++++++++++--- packages/plugin/test/demandLadder.test.js | 38 +++++++++++++++++ packages/plugin/test/routeClass.test.js | 45 ++++++++++++++++++++ 7 files changed, 161 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index f5056546..52af2e2c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.54.0", + "version": "0.55.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 1b5eb97b..a09c00af 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.54.0", + "version": "0.55.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index fee3df5c..4e6a761b 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -104,7 +104,7 @@ export const configSchema = group('Prerender plugin configuration.', { [], 'Ordered route list (forwarded mode). Each entry is ' + "{ match: 'exact' | 'prefix' | 'contains', path: string, mode?: 'prerender' | 'passthrough', " + - 'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean }.\n\n' + + 'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean, demandFloor?: number }.\n\n' + 'FIRST MATCH WINS, so order most-specific first. That ordering is what lets a passthrough ' + 'carve-out sit inside a prerendered prefix (`/products/clearance/` above `/products/`) ' + 'without a second list and a precedence rule.\n\n' + @@ -129,6 +129,15 @@ export const configSchema = group('Prerender plugin configuration.', { 'must be kept aligned BY HAND — rendering much faster than the edge TTL burns renders the edge ' + 'never serves, and much slower means the edge re-fetches stale content. Neither side can see ' + 'the other drift.\n\n' + + '`demandFloor` (ms, prerender routes only) — the FASTEST cadence the demand ladder ' + + "(`render.demand`) may grant this route's pages: rungs faster than the floor are unreachable " + + 'for them. This is how a deployment keeps fast global rungs for a corpus that earns them ' + + '(listing pages that churn intraday) without a breadth-sweeping crawler promoting a much ' + + 'larger corpus onto the same rungs — a crawler that recrawls EVERYTHING daily makes ' + + '"visited" true everywhere, and without a floor the ladder would grant the whole route the ' + + 'fast cadence at corpus scale. A floor at or above the granted cadence leaves the route ' + + 'resting at that cadence (single-rung). Stored rungs below a newly-raised floor read as the ' + + 'floor immediately and re-stamp on their next ladder decision. Live, like renderInterval.\n\n' + '`discoverTargets` (default true, prerender routes only) — whether a bot visiting an UNKNOWN ' + 'URL on this route creates a target for it. Set false on routes whose URL space is ' + 'combinatorial (faceted navigation, filter/sort permutations): crawlers walking those links ' + @@ -980,7 +989,9 @@ export const configSchema = group('Prerender plugin configuration.', { '6h, or a weekly sitemap route pulled to 48h at 3.5x its granted render budget). ' + 'Bottoming out at 6h rather than 1h is deliberate: 1h buys ' + '~0.04% availability error against 6h\u2019s ~0.24% for six times the render cost, and the ' + - 'fast rungs are where a runaway hot set becomes unaffordable.', + 'fast rungs are where a runaway hot set becomes unaffordable. A route can bound how much ' + + 'of this ladder its pages may use with `ingress.routes[].demandFloor` \u2014 rungs faster than ' + + 'a route\u2019s floor are unreachable for that route.', { unit: 'ms' } ), promoteWindows: option( diff --git a/packages/plugin/src/util/demandLadder.js b/packages/plugin/src/util/demandLadder.js index 7641797d..0b5a98b2 100644 --- a/packages/plugin/src/util/demandLadder.js +++ b/packages/plugin/src/util/demandLadder.js @@ -51,6 +51,7 @@ import { config, onConfigApplied } from '../config.js'; import { visitedWithin, visitedInEachWindow, mergedReady, ensureMerged, newestFill } from './visitFilter.js'; +import { demandFloorFor } from './routeClass.js'; import { metrics } from '../metrics.js'; // Normalized rungs, recomputed only when config changes — decideInterval runs on the @@ -71,7 +72,11 @@ export const rungs = () => cachedRungs; /** * The ladder a given base interval actually offers: every configured rung STRICTLY faster - * than base, with base itself as the top (slowest) rung. + * than base, with base itself as the top (slowest) rung. A route's `demandFloor` (second + * argument) removes the rungs faster than it — the floor is the fastest cadence that route's + * pages may be granted, so a deployment can keep fast global rungs for one route's corpus + * without a breadth-sweeping crawler dragging another route onto them. A floor at or above + * the base leaves a single-rung ladder: the route rests at its granted cadence. * * Base is the resting state, and an off-ladder base is deliberately never snapped to a * configured rung — in either direction. Snapping UP schedules slower than the route granted @@ -85,12 +90,16 @@ export const rungs = () => cachedRungs; * Memoized per base: bases come from routes/changefreq/default — a handful of distinct * values — and this runs per reschedule. Bounded anyway so a caller bug cannot grow it. */ -export const effectiveLadder = (base) => { - let list = effectiveLadders.get(base); +export const effectiveLadder = (base, floor = null) => { + // Composite key: bases and floors both come from a handful of route/config values, so the + // memo stays single-digit; a route's floor CHANGE arrives as a different key, never as + // stale content under an old one. + const key = `${base}|${floor ?? ''}`; + let list = effectiveLadders.get(key); if (!list) { if (effectiveLadders.size >= 64) effectiveLadders.clear(); - list = [...cachedRungs.filter((r) => r < base), base]; - effectiveLadders.set(base, list); + list = [...cachedRungs.filter((r) => r < base && (floor === null || r >= floor)), base]; + effectiveLadders.set(key, list); } return list; }; @@ -171,7 +180,10 @@ export function decideInterval(url, base, current, nowMs = Date.now(), probe = v // granted cadence — nothing to reallocate, so skip the probe (and the cold hold: there // is no move to hold back). Counted apart from `held`: the ladder did not hold this // target at its rung, it was never offered another one. - const list = effectiveLadder(Number(base)); + // The matched route's demandFloor bounds how fast this URL may be granted. Derived here + // rather than threaded from the caller so every decision site gets it for free; classify + // is a memoized-compile linear scan over a handful of routes, on a ~20x/s path. + const list = effectiveLadder(Number(base), demandFloorFor(url)); if (list.length === 1) { stats.singleRung++; return { interval: base, level: base, action: 'single-rung' }; diff --git a/packages/plugin/src/util/routeClass.js b/packages/plugin/src/util/routeClass.js index 77e0698e..2dfbfd89 100644 --- a/packages/plugin/src/util/routeClass.js +++ b/packages/plugin/src/util/routeClass.js @@ -122,7 +122,26 @@ const compileEntry = (raw, source, warn) => { } } - return { match: raw.match, path: raw.path, mode, queryParams, renderInterval, discoverTargets, source }; + // Optional per-route demand-ladder floor (ms) — the FASTEST cadence the ladder may grant + // this route's pages (see util/demandLadder.js). Same drop-the-FIELD rule as the two above. + let demandFloor = null; + if (raw.demandFloor !== undefined && raw.demandFloor !== null) { + if (mode === PASSTHROUGH) { + warn( + `ignoring demandFloor on passthrough route "${raw.match} ${raw.path}" — a passthrough route is ` + + `never scheduled, so it has no cadence to floor` + ); + } else if (Number.isFinite(raw.demandFloor) && raw.demandFloor > 0) { + demandFloor = raw.demandFloor; + } else { + warn( + `ignoring demandFloor on route "${raw.match} ${raw.path}" — expected a positive number of ` + + `milliseconds, got ${String(raw.demandFloor)}` + ); + } + } + + return { match: raw.match, path: raw.path, mode, queryParams, renderInterval, discoverTargets, demandFloor, source }; }; /** @@ -402,8 +421,9 @@ export const queryAllowlistFor = (rawUrl) => classifyUrl(rawUrl).queryParams; * Same contract as `classifyUrl`: `url` is a device-free public URL (the Target primary key * / the url-half of a cacheKey). */ -export const resolveRenderInterval = (url, storedInterval) => { - const { entry } = classifyUrl(url); +export const resolveRenderInterval = (url, storedInterval) => baseInterval(classifyUrl(url).entry, storedInterval); + +const baseInterval = (entry, storedInterval) => { if (entry && entry.renderInterval !== null && entry.renderInterval !== undefined) return entry.renderInterval; // Coerce before the finite check: `Long` columns can surface the stored interval as a // BigInt, which `Number.isFinite` rejects outright — without this, every such target @@ -413,6 +433,18 @@ export const resolveRenderInterval = (url, storedInterval) => { return Number.isFinite(stored) && stored > 0 ? stored : config.render.defaultInterval; }; +const entryFloor = (entry) => { + const floor = entry ? Number(entry.demandFloor) : NaN; + return Number.isFinite(floor) && floor > 0 ? floor : null; +}; + +/** + * The demand-ladder floor the matched route sets for a URL, or null (no floor). The ladder + * consults this per decision (`util/demandLadder.js#decideInterval`), so a route's floor is + * live config like its `renderInterval` — no data migration on change. + */ +export const demandFloorFor = (url) => entryFloor(classifyUrl(url).entry); + /** * The cadence a target's `nextRenderTime` is ACTUALLY computed from — `resolveRenderInterval` * with the demand ladder's current rung applied on top. @@ -430,12 +462,20 @@ export const resolveRenderInterval = (url, storedInterval) => { * its next render. `decideInterval` clamps the same way (`Math.min(rungIndexOf(...), ceiling)`), so * a stale rung reads as "at the ceiling" here exactly as it does there rather than understating the * cadence. + * + * CLAMPED TO THE ROUTE'S `demandFloor` for the same staleness reason in the other direction: + * raise a route's floor and every target below it still carries its old, faster rung until the + * ladder re-decides. The ladder will snap it into the floored list on its next decision; until + * then this must not credit a cadence the ladder no longer grants. */ export const resolveEffectiveInterval = (url, { renderInterval, demandInterval } = {}) => { - const base = resolveRenderInterval(url, renderInterval); + const { entry } = classifyUrl(url); + const base = baseInterval(entry, renderInterval); // Same BigInt-from-`Long` coercion as above, and the same reason `> 0` rather than a typeof // check: `Number(null)` is 0 and `Number(undefined)` is NaN, so an unevaluated target — which // is most of the corpus until the ladder reaches it — falls through to the ceiling. const rung = Number(demandInterval); - return Number.isFinite(rung) && rung > 0 ? Math.min(rung, base) : base; + if (!(Number.isFinite(rung) && rung > 0)) return base; + const floor = entryFloor(entry); + return Math.min(floor !== null ? Math.max(rung, floor) : rung, base); }; diff --git a/packages/plugin/test/demandLadder.test.js b/packages/plugin/test/demandLadder.test.js index 85551c9c..4694e13b 100644 --- a/packages/plugin/test/demandLadder.test.js +++ b/packages/plugin/test/demandLadder.test.js @@ -285,3 +285,41 @@ test('config defaults ship inert: enabled off and dryRun on', () => { assert.equal(d.render.demand.dryRun, true); assert.equal(config.render.demand.enabled, true, 'test override applied on top of defaults'); }); + +test('a route demandFloor bounds promotion, heals sub-floor stamps, and leaves other routes alone', () => { + const T = 1_700_000_000_000; + applyOptions({ + render: { + demand: { + enabled: true, + dryRun: false, + ladder: [6 * H, 12 * H, 24 * H, 48 * H], + promoteWindows: 2, + maxFastInterval: 12 * H, + maxFastFraction: 0.05, + }, + }, + ingress: { + mode: 'forwarded', + routes: [ + { match: 'prefix', path: '/product/prd-', renderInterval: 96 * H, demandFloor: 48 * H }, + { match: 'prefix', path: '/catalog/', renderInterval: 24 * H, demandFloor: 48 * H }, + ], + }, + }); + const pdp = 'https://www.example.com/product/prd-1'; + // Promotion stops at the floor: the only rung under the 96h base is 48h. + assert.deepEqual(decideInterval(pdp, 96 * H, undefined, T, always), { + interval: 48 * H, + level: 48 * H, + action: 'promoted', + }); + // A stored rung below the floor snaps INTO the floored list and cannot promote past it. + const healed = decideInterval(pdp, 96 * H, 6 * H, T, always); + assert.equal(healed.level, 48 * H); + assert.equal(healed.action, 'held'); + // A floor at or above the granted cadence leaves a single-rung ladder: rest at the grant. + assert.equal(decideInterval('https://www.example.com/catalog/x', 24 * H, undefined, T, always).action, 'single-rung'); + // A URL matching no route keeps the full ladder. + assert.equal(decideInterval('https://www.example.com/other', 96 * H, 48 * H, T, always).interval, 24 * H); +}); diff --git a/packages/plugin/test/routeClass.test.js b/packages/plugin/test/routeClass.test.js index a09f42cc..f2fefe32 100644 --- a/packages/plugin/test/routeClass.test.js +++ b/packages/plugin/test/routeClass.test.js @@ -9,6 +9,7 @@ import { queryAllowlistFor, resolveEffectiveInterval, resolveRenderInterval, + demandFloorFor, PASSTHROUGH, PRERENDER, UNCLASSIFIED, @@ -352,3 +353,47 @@ test('route discoverTargets defaults to true and is ignored on a passthrough rou // A passthrough route never schedules, so the field is warned about and ignored. assert.equal(matchRoute('/help/contact-us').discoverTargets, true); }); + +test('route demandFloor: valid values kept, invalid values drop the FIELD but never the route', () => { + forwarded({ + ingress: { + routes: [ + { match: 'prefix', path: '/catalog/', demandFloor: 12 * HOUR_MS }, + { match: 'prefix', path: '/product/prd-', demandFloor: 'fast' }, // invalid → field dropped + { match: 'prefix', path: '/help/', mode: 'passthrough', demandFloor: HOUR_MS }, // ignored + ], + }, + }); + assert.equal(matchRoute('/catalog/x').demandFloor, 12 * HOUR_MS); + assert.equal(classifyPath('/product/prd-1').routeClass, PRERENDER); + assert.equal(matchRoute('/product/prd-1').demandFloor, null); + assert.equal(matchRoute('/help/contact-us').demandFloor, null); + assert.equal(demandFloorFor('https://www.example.com/catalog/x'), 12 * HOUR_MS); + assert.equal(demandFloorFor('https://www.example.com/product/prd-1'), null); + assert.equal(demandFloorFor('https://www.example.com/unrouted'), null); +}); + +test('resolveEffectiveInterval clamps a stored rung UP to the route demandFloor', () => { + const DAY_MS = 24 * HOUR_MS; + forwarded({ + ingress: { + routes: [ + { match: 'prefix', path: '/catalog/', renderInterval: DAY_MS, demandFloor: 12 * HOUR_MS }, + { match: 'prefix', path: '/product/prd-', renderInterval: 4 * DAY_MS }, + ], + }, + }); + const catalog = 'https://www.example.com/catalog/x'; + // A stale sub-floor stamp must not be credited: reads as the floor until the ladder re-stamps. + assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 6 * HOUR_MS }), 12 * HOUR_MS); + // At or above the floor, the rung stands (still ceiling-clamped). + assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 12 * HOUR_MS }), 12 * HOUR_MS); + assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 3 * DAY_MS }), DAY_MS); + // No floor on the route → unchanged behavior. + assert.equal( + resolveEffectiveInterval('https://www.example.com/product/prd-1', { demandInterval: 6 * HOUR_MS }), + 6 * HOUR_MS + ); + // Unset rung stays at the ceiling regardless of the floor. + assert.equal(resolveEffectiveInterval(catalog, {}), DAY_MS); +});