feat(plugin): per-route demandFloor — bound the demand ladder per route; v0.55.0

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.54.0",
+	"version": "0.55.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/configSchema.js

@@ -104,7 +104,7 @@ export const configSchema = group('Prerender plugin configuration.', {
 				[],
 				'Ordered route list (forwarded mode). Each entry is ' +
 					"{ match: 'exact' | 'prefix' | 'contains', path: string, mode?: 'prerender' | 'passthrough', " +
-					'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean }.\n\n' +
+					'queryParams?: string[], renderInterval?: number, discoverTargets?: boolean, demandFloor?: number }.\n\n' +
 					'FIRST MATCH WINS, so order most-specific first. That ordering is what lets a passthrough ' +
 					'carve-out sit inside a prerendered prefix (`/products/clearance/` above `/products/`) ' +
 					'without a second list and a precedence rule.\n\n' +
@@ -129,6 +129,15 @@ export const configSchema = group('Prerender plugin configuration.', {
 					'must be kept aligned BY HAND — rendering much faster than the edge TTL burns renders the edge ' +
 					'never serves, and much slower means the edge re-fetches stale content. Neither side can see ' +
 					'the other drift.\n\n' +
+					'`demandFloor` (ms, prerender routes only) — the FASTEST cadence the demand ladder ' +
+					"(`render.demand`) may grant this route's pages: rungs faster than the floor are unreachable " +
+					'for them. This is how a deployment keeps fast global rungs for a corpus that earns them ' +
+					'(listing pages that churn intraday) without a breadth-sweeping crawler promoting a much ' +
+					'larger corpus onto the same rungs — a crawler that recrawls EVERYTHING daily makes ' +
+					'"visited" true everywhere, and without a floor the ladder would grant the whole route the ' +
+					'fast cadence at corpus scale. A floor at or above the granted cadence leaves the route ' +
+					'resting at that cadence (single-rung). Stored rungs below a newly-raised floor read as the ' +
+					'floor immediately and re-stamp on their next ladder decision. Live, like renderInterval.\n\n' +
 					'`discoverTargets` (default true, prerender routes only) — whether a bot visiting an UNKNOWN ' +
 					'URL on this route creates a target for it. Set false on routes whose URL space is ' +
 					'combinatorial (faceted navigation, filter/sort permutations): crawlers walking those links ' +
@@ -980,7 +989,9 @@ export const configSchema = group('Prerender plugin configuration.', {
 						'6h, or a weekly sitemap route pulled to 48h at 3.5x its granted render budget). ' +
 						'Bottoming out at 6h rather than 1h is deliberate: 1h buys ' +
 						'~0.04% availability error against 6h\u2019s ~0.24% for six times the render cost, and the ' +
-						'fast rungs are where a runaway hot set becomes unaffordable.',
+						'fast rungs are where a runaway hot set becomes unaffordable. A route can bound how much ' +
+						'of this ladder its pages may use with `ingress.routes[].demandFloor` \u2014 rungs faster than ' +
+						'a route\u2019s floor are unreachable for that route.',
 					{ unit: 'ms' }
 				),
 				promoteWindows: option(

packages/plugin/src/util/demandLadder.js

@@ -51,6 +51,7 @@
 
 import { config, onConfigApplied } from '../config.js';
 import { visitedWithin, visitedInEachWindow, mergedReady, ensureMerged, newestFill } from './visitFilter.js';
+import { demandFloorFor } from './routeClass.js';
 import { metrics } from '../metrics.js';
 
 // Normalized rungs, recomputed only when config changes — decideInterval runs on the
@@ -71,7 +72,11 @@ export const rungs = () => cachedRungs;
 
 /**
  * The ladder a given base interval actually offers: every configured rung STRICTLY faster
- * than base, with base itself as the top (slowest) rung.
+ * than base, with base itself as the top (slowest) rung. A route's `demandFloor` (second
+ * argument) removes the rungs faster than it — the floor is the fastest cadence that route's
+ * pages may be granted, so a deployment can keep fast global rungs for one route's corpus
+ * without a breadth-sweeping crawler dragging another route onto them. A floor at or above
+ * the base leaves a single-rung ladder: the route rests at its granted cadence.
  *
  * Base is the resting state, and an off-ladder base is deliberately never snapped to a
  * configured rung — in either direction. Snapping UP schedules slower than the route granted
@@ -85,12 +90,16 @@ export const rungs = () => cachedRungs;
  * Memoized per base: bases come from routes/changefreq/default — a handful of distinct
  * values — and this runs per reschedule. Bounded anyway so a caller bug cannot grow it.
  */
-export const effectiveLadder = (base) => {
-	let list = effectiveLadders.get(base);
+export const effectiveLadder = (base, floor = null) => {
+	// Composite key: bases and floors both come from a handful of route/config values, so the
+	// memo stays single-digit; a route's floor CHANGE arrives as a different key, never as
+	// stale content under an old one.
+	const key = `${base}|${floor ?? ''}`;
+	let list = effectiveLadders.get(key);
 	if (!list) {
 		if (effectiveLadders.size >= 64) effectiveLadders.clear();
-		list = [...cachedRungs.filter((r) => r < base), base];
-		effectiveLadders.set(base, list);
+		list = [...cachedRungs.filter((r) => r < base && (floor === null || r >= floor)), base];
+		effectiveLadders.set(key, list);
 	}
 	return list;
 };
@@ -171,7 +180,10 @@ export function decideInterval(url, base, current, nowMs = Date.now(), probe = v
 	// granted cadence — nothing to reallocate, so skip the probe (and the cold hold: there
 	// is no move to hold back). Counted apart from `held`: the ladder did not hold this
 	// target at its rung, it was never offered another one.
-	const list = effectiveLadder(Number(base));
+	// The matched route's demandFloor bounds how fast this URL may be granted. Derived here
+	// rather than threaded from the caller so every decision site gets it for free; classify
+	// is a memoized-compile linear scan over a handful of routes, on a ~20x/s path.
+	const list = effectiveLadder(Number(base), demandFloorFor(url));
 	if (list.length === 1) {
 		stats.singleRung++;
 		return { interval: base, level: base, action: 'single-rung' };

packages/plugin/src/util/routeClass.js

@@ -122,7 +122,26 @@ const compileEntry = (raw, source, warn) => {
 		}
 	}
 
-	return { match: raw.match, path: raw.path, mode, queryParams, renderInterval, discoverTargets, source };
+	// Optional per-route demand-ladder floor (ms) — the FASTEST cadence the ladder may grant
+	// this route's pages (see util/demandLadder.js). Same drop-the-FIELD rule as the two above.
+	let demandFloor = null;
+	if (raw.demandFloor !== undefined && raw.demandFloor !== null) {
+		if (mode === PASSTHROUGH) {
+			warn(
+				`ignoring demandFloor on passthrough route "${raw.match} ${raw.path}" — a passthrough route is ` +
+					`never scheduled, so it has no cadence to floor`
+			);
+		} else if (Number.isFinite(raw.demandFloor) && raw.demandFloor > 0) {
+			demandFloor = raw.demandFloor;
+		} else {
+			warn(
+				`ignoring demandFloor on route "${raw.match} ${raw.path}" — expected a positive number of ` +
+					`milliseconds, got ${String(raw.demandFloor)}`
+			);
+		}
+	}
+
+	return { match: raw.match, path: raw.path, mode, queryParams, renderInterval, discoverTargets, demandFloor, source };
 };
 
 /**
@@ -402,8 +421,9 @@ export const queryAllowlistFor = (rawUrl) => classifyUrl(rawUrl).queryParams;
  * Same contract as `classifyUrl`: `url` is a device-free public URL (the Target primary key
  * / the url-half of a cacheKey).
  */
-export const resolveRenderInterval = (url, storedInterval) => {
-	const { entry } = classifyUrl(url);
+export const resolveRenderInterval = (url, storedInterval) => baseInterval(classifyUrl(url).entry, storedInterval);
+
+const baseInterval = (entry, storedInterval) => {
 	if (entry && entry.renderInterval !== null && entry.renderInterval !== undefined) return entry.renderInterval;
 	// Coerce before the finite check: `Long` columns can surface the stored interval as a
 	// BigInt, which `Number.isFinite` rejects outright — without this, every such target
@@ -413,6 +433,18 @@ export const resolveRenderInterval = (url, storedInterval) => {
 	return Number.isFinite(stored) && stored > 0 ? stored : config.render.defaultInterval;
 };
 
+const entryFloor = (entry) => {
+	const floor = entry ? Number(entry.demandFloor) : NaN;
+	return Number.isFinite(floor) && floor > 0 ? floor : null;
+};
+
+/**
+ * The demand-ladder floor the matched route sets for a URL, or null (no floor). The ladder
+ * consults this per decision (`util/demandLadder.js#decideInterval`), so a route's floor is
+ * live config like its `renderInterval` — no data migration on change.
+ */
+export const demandFloorFor = (url) => entryFloor(classifyUrl(url).entry);
+
 /**
  * The cadence a target's `nextRenderTime` is ACTUALLY computed from — `resolveRenderInterval`
  * with the demand ladder's current rung applied on top.
@@ -430,12 +462,20 @@ export const resolveRenderInterval = (url, storedInterval) => {
  * its next render. `decideInterval` clamps the same way (`Math.min(rungIndexOf(...), ceiling)`), so
  * a stale rung reads as "at the ceiling" here exactly as it does there rather than understating the
  * cadence.
+ *
+ * CLAMPED TO THE ROUTE'S `demandFloor` for the same staleness reason in the other direction:
+ * raise a route's floor and every target below it still carries its old, faster rung until the
+ * ladder re-decides. The ladder will snap it into the floored list on its next decision; until
+ * then this must not credit a cadence the ladder no longer grants.
  */
 export const resolveEffectiveInterval = (url, { renderInterval, demandInterval } = {}) => {
-	const base = resolveRenderInterval(url, renderInterval);
+	const { entry } = classifyUrl(url);
+	const base = baseInterval(entry, renderInterval);
 	// Same BigInt-from-`Long` coercion as above, and the same reason `> 0` rather than a typeof
 	// check: `Number(null)` is 0 and `Number(undefined)` is NaN, so an unevaluated target — which
 	// is most of the corpus until the ladder reaches it — falls through to the ceiling.
 	const rung = Number(demandInterval);
-	return Number.isFinite(rung) && rung > 0 ? Math.min(rung, base) : base;
+	if (!(Number.isFinite(rung) && rung > 0)) return base;
+	const floor = entryFloor(entry);
+	return Math.min(floor !== null ? Math.max(rung, floor) : rung, base);
 };

packages/plugin/test/demandLadder.test.js

@@ -285,3 +285,41 @@ test('config defaults ship inert: enabled off and dryRun on', () => {
 	assert.equal(d.render.demand.dryRun, true);
 	assert.equal(config.render.demand.enabled, true, 'test override applied on top of defaults');
 });
+
+test('a route demandFloor bounds promotion, heals sub-floor stamps, and leaves other routes alone', () => {
+	const T = 1_700_000_000_000;
+	applyOptions({
+		render: {
+			demand: {
+				enabled: true,
+				dryRun: false,
+				ladder: [6 * H, 12 * H, 24 * H, 48 * H],
+				promoteWindows: 2,
+				maxFastInterval: 12 * H,
+				maxFastFraction: 0.05,
+			},
+		},
+		ingress: {
+			mode: 'forwarded',
+			routes: [
+				{ match: 'prefix', path: '/product/prd-', renderInterval: 96 * H, demandFloor: 48 * H },
+				{ match: 'prefix', path: '/catalog/', renderInterval: 24 * H, demandFloor: 48 * H },
+			],
+		},
+	});
+	const pdp = 'https://www.example.com/product/prd-1';
+	// Promotion stops at the floor: the only rung under the 96h base is 48h.
+	assert.deepEqual(decideInterval(pdp, 96 * H, undefined, T, always), {
+		interval: 48 * H,
+		level: 48 * H,
+		action: 'promoted',
+	});
+	// A stored rung below the floor snaps INTO the floored list and cannot promote past it.
+	const healed = decideInterval(pdp, 96 * H, 6 * H, T, always);
+	assert.equal(healed.level, 48 * H);
+	assert.equal(healed.action, 'held');
+	// A floor at or above the granted cadence leaves a single-rung ladder: rest at the grant.
+	assert.equal(decideInterval('https://www.example.com/catalog/x', 24 * H, undefined, T, always).action, 'single-rung');
+	// A URL matching no route keeps the full ladder.
+	assert.equal(decideInterval('https://www.example.com/other', 96 * H, 48 * H, T, always).interval, 24 * H);
+});

packages/plugin/test/routeClass.test.js

@@ -9,6 +9,7 @@ import {
 	queryAllowlistFor,
 	resolveEffectiveInterval,
 	resolveRenderInterval,
+	demandFloorFor,
 	PASSTHROUGH,
 	PRERENDER,
 	UNCLASSIFIED,
@@ -352,3 +353,47 @@ test('route discoverTargets defaults to true and is ignored on a passthrough rou
 	// A passthrough route never schedules, so the field is warned about and ignored.
 	assert.equal(matchRoute('/help/contact-us').discoverTargets, true);
 });
+
+test('route demandFloor: valid values kept, invalid values drop the FIELD but never the route', () => {
+	forwarded({
+		ingress: {
+			routes: [
+				{ match: 'prefix', path: '/catalog/', demandFloor: 12 * HOUR_MS },
+				{ match: 'prefix', path: '/product/prd-', demandFloor: 'fast' }, // invalid → field dropped
+				{ match: 'prefix', path: '/help/', mode: 'passthrough', demandFloor: HOUR_MS }, // ignored
+			],
+		},
+	});
+	assert.equal(matchRoute('/catalog/x').demandFloor, 12 * HOUR_MS);
+	assert.equal(classifyPath('/product/prd-1').routeClass, PRERENDER);
+	assert.equal(matchRoute('/product/prd-1').demandFloor, null);
+	assert.equal(matchRoute('/help/contact-us').demandFloor, null);
+	assert.equal(demandFloorFor('https://www.example.com/catalog/x'), 12 * HOUR_MS);
+	assert.equal(demandFloorFor('https://www.example.com/product/prd-1'), null);
+	assert.equal(demandFloorFor('https://www.example.com/unrouted'), null);
+});
+
+test('resolveEffectiveInterval clamps a stored rung UP to the route demandFloor', () => {
+	const DAY_MS = 24 * HOUR_MS;
+	forwarded({
+		ingress: {
+			routes: [
+				{ match: 'prefix', path: '/catalog/', renderInterval: DAY_MS, demandFloor: 12 * HOUR_MS },
+				{ match: 'prefix', path: '/product/prd-', renderInterval: 4 * DAY_MS },
+			],
+		},
+	});
+	const catalog = 'https://www.example.com/catalog/x';
+	// A stale sub-floor stamp must not be credited: reads as the floor until the ladder re-stamps.
+	assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 6 * HOUR_MS }), 12 * HOUR_MS);
+	// At or above the floor, the rung stands (still ceiling-clamped).
+	assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 12 * HOUR_MS }), 12 * HOUR_MS);
+	assert.equal(resolveEffectiveInterval(catalog, { demandInterval: 3 * DAY_MS }), DAY_MS);
+	// No floor on the route → unchanged behavior.
+	assert.equal(
+		resolveEffectiveInterval('https://www.example.com/product/prd-1', { demandInterval: 6 * HOUR_MS }),
+		6 * HOUR_MS
+	);
+	// Unset rung stays at the ceiling regardless of the floor.
+	assert.equal(resolveEffectiveInterval(catalog, {}), DAY_MS);
+});