fix(plugin): let sitemap departure run uncapped, and say that capped departures are dropped for good; v0.87.0

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.86.0",
+	"version": "0.87.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/configSchema.js

@@ -2082,23 +2082,47 @@ export const configSchema = group('Prerender plugin configuration.', {
 						'looked, and one that decides whether `maxActions` is a ceiling or a no-op. The result ' +
 						'and the progress row carry the outcome tally either way.'
 				),
+				// -1 is "no ceiling" for both caps below, and it is -1 rather than `null` or `Infinity` on
+				// purpose. `null` already means "no value" in every layer — the merge skips it and keeps the
+				// default, an override write refuses it, and the console's number field reports it for an
+				// emptied box — so making it mean "uncapped" would turn a blank `maxActions:` in config.yaml
+				// from the default into no guard at all, silently and in the dangerous direction. `Infinity`
+				// cannot cross JSON, so no override row or console edit could ever carry it. -1 is a plain
+				// number every layer already validates and transports, and nobody types it by accident.
+				// 0 keeps its meaning — act on / collect nothing — because 0 is also how a walk with no
+				// opted-in route switches collection off. Read them through `departureLimit`.
 				maxActions: option(
 					5000,
-					'Ceiling on departed URLs ACTED ON in one walk. Skipped candidates (re-attached, ' +
-						'suppressed, route opted out) do not count against it. The cap is the guard against a ' +
-						'pathological walk: a child sitemap that fetches truncated but still parses as valid XML ' +
-						'presents every URL it no longer lists as departed, and without a ceiling one bad fetch ' +
-						'would expire a large slice of the cache. Overflow is counted as `capped` and left for ' +
-						'the next walk rather than silently dropped.',
-					{ min: 0 }
+					'Ceiling on departed URLs ACTED ON in one walk, or -1 for no ceiling. Skipped candidates ' +
+						'(re-attached, suppressed, route opted out) do not count against it.\n\n' +
+						'WHAT IT GUARDS AGAINST: a child sitemap that fetches truncated but still parses as valid ' +
+						'XML presents every URL it no longer lists as departed, and without a ceiling that one bad ' +
+						'fetch would expire a large slice of the cache.\n\n' +
+						'OVERFLOW IS DROPPED FOR GOOD, not deferred. By the time the check runs the walk has already ' +
+						'unlinked the URL from its sitemap, and a later walk only looks for departures among URLs ' +
+						'still linked to one — so a URL counted as `capped` is never offered again (unless it ' +
+						're-enters a sitemap and departs a second time). It keeps its target, its render cadence ' +
+						'and its cached page; it just never gets its route’s departure action. Read `capped` as ' +
+						'departures lost, not postponed.\n\n' +
+						'-1 removes the ceiling: every departed URL gets its action and nothing is ever counted as ' +
+						'`capped`. That gives up the guard above, so read a `dryRun` tally first. 0 acts on ' +
+						'nothing — every candidate is still decided and reported.',
+					{ min: -1 }
 				),
 				maxCandidates: option(
 					50000,
-					'Ceiling on departed URLs HELD for the post-walk check. Separate from `maxActions` because ' +
-						'this one bounds memory: it is a list of URLs retained across a walk that can prune ' +
-						'millions. A capped list is reported as such, so a short list is never presented as ' +
-						'"few departed".',
-					{ min: 0 }
+					'Ceiling on departed URLs HELD for the post-walk check, or -1 for no ceiling. Separate from ' +
+						'`maxActions` because this one bounds memory: it is a list of URLs retained across a walk ' +
+						'that can prune millions. Uncapped, the list is bounded by the URLs that walk actually ' +
+						'unlinks — at most `scan.collectCap` per child sitemap — so it grows with how much of the ' +
+						'corpus departs at once, which is exactly what a truncated child sitemap inflates. A capped ' +
+						'list is reported as such (`departures.capped` on the result and the progress row), so a ' +
+						'short list is never presented as "few departed".\n\n' +
+						'Overflow is DROPPED FOR GOOD, for the same reason as `maxActions` overflow: the URL is ' +
+						'already unlinked, so no later walk offers it again. It is never decided and never appears ' +
+						'in the outcome tally; `removed` minus `departures.considered` is how many were lost. -1 ' +
+						'removes the ceiling and never reports `capped`; 0 collects nothing.',
+					{ min: -1 }
 				),
 			}
 		),

packages/plugin/src/resources/Sitemap.js

@@ -2,15 +2,15 @@ import { config, onConfigApplied } from '../config.js';
 import { metrics } from '../metrics.js';
 import { describeError } from '../util/errors.js';
 import { Target } from './Target.js';
-import { anyRouteDeparts, classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js';
+import { classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js';
 import { currentMinuteMs, epochMsOf, getInitialRenderTime, getNextSitemapRefreshTime } from '../util/time.js';
 import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js';
 import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js';
 import { configuredStagingIp, dispatcherFor } from '../util/upstream.js';
 import { setImmediate } from 'node:timers/promises';
 import { applyInBatches, collectFromScan } from '../util/scan.js';
 import { conditionalValidatorFor } from '../util/sitemapConditional.js';
-import { decideDeparture, DepartureAction } from '../util/sitemapDeparture.js';
+import { decideDeparture, DepartureAction, departureCandidateCap, departureLimit } from '../util/sitemapDeparture.js';
 import { cacheKeysOf } from './Target.js';
 import { writeSchedule } from '../util/renderSchedule.js';
 import { resolveEffectiveInterval } from '../util/routeClass.js';
@@ -58,10 +58,9 @@ class Sitemap extends databases.sitemaps.Sitemap {
 		const run = createRefreshRun({
 			removedSampleCap: config.sitemap.removedSampleCap,
 			failedCap: config.sitemap.failedCap,
-			// Zero unless the departure check is on AND some route opts in, which keeps the candidate
-			// list empty — and `addRemoved` allocation-free — for every deployment that has not asked
-			// for this. `anyRouteDeparts` is resolved once per walk rather than per departed URL.
-			departureCap: config.sitemap.departure.enabled && anyRouteDeparts() ? config.sitemap.departure.maxCandidates : 0,
+			// Zero unless the departure check is on AND some route opts in; Infinity when
+			// `maxCandidates` is -1. See util/sitemapDeparture.js.
+			departureCap: departureCandidateCap(),
 		});
 		const visited = new Set();
 		const queue = [{ url: rootSitemapUrl, parentUrl: null }];
@@ -750,12 +749,19 @@ function getTtlFromChangeFreq(changefreq, { minTtl, defaultTtl }) {
  * Every candidate is decided and counted, including the ones nothing happens to, so a dry run
  * answers the question a deployment actually has before enabling this: how many URLs a real walk
  * would touch, and how many of the departures are shear rather than departure.
+ *
+ * A candidate refused by `maxActions` is counted `capped` and is gone for good — the walk already
+ * unlinked it, so no later walk offers it again. `maxActions: -1` removes the ceiling.
+ *
+ * Exported for tests.
  */
-async function processDepartures(run) {
+export async function processDepartures(run) {
 	const urls = run.departureCandidates();
 	if (!urls.length) return;
 
-	const { dryRun, maxActions } = config.sitemap.departure;
+	const { dryRun } = config.sitemap.departure;
+	// Through `departureLimit`, never raw: -1 is "no ceiling", and `acted >= -1` would refuse all.
+	const maxActions = departureLimit(config.sitemap.departure.maxActions);
 	let acted = 0;
 
 	await applyInBatches({
@@ -827,7 +833,8 @@ async function processDepartures(run) {
 		.map(([name, count]) => `${name} ${count}`)
 		.join(', ');
 	logger.info(
-		`[prerender] Departure check: ${considered} candidates${capped ? ' (CAPPED — some departed URLs were never checked)' : ''}` +
+		`[prerender] Departure check: ${considered} candidates` +
+			`${capped ? ' (CAPPED at maxCandidates — the departed URLs past it were never checked, and no later walk will offer them)' : ''}` +
 			`${dryRun ? ', DRY RUN' : ''} — ${summary || 'nothing to do'}`
 	);
 

packages/plugin/src/util/sitemapDeparture.js

@@ -1,5 +1,5 @@
 import { config } from '../config.js';
-import { classifyUrl, DepartureAction, PRERENDER } from './routeClass.js';
+import { anyRouteDeparts, classifyUrl, DepartureAction, PRERENDER } from './routeClass.js';
 
 /**
  * What a URL LEAVING a sitemap means, and what to do about it.
@@ -39,13 +39,44 @@ import { classifyUrl, DepartureAction, PRERENDER } from './routeClass.js';
  * Candidates are collected during the walk and re-read once it has finished; anything that picked
  * up an attribution in the meantime is dropped. That is also why this cannot be folded into
  * `reconcileSitemapEntries`.
+ *
+ * ── WHY A CAPPED DEPARTURE IS GONE FOR GOOD ──────────────────────────────────────────────────
+ *
+ * The prune unlinks a departed target (`sitemapUrl -> null`) BEFORE it becomes a candidate, and a
+ * later walk finds departures by scanning for targets still linked to the sitemap being pruned. So
+ * a URL that overflows either ceiling — past `maxCandidates` it is never held, past `maxActions` it
+ * is decided but not acted on — is not left for the next walk: no walk will offer it again unless
+ * it re-enters a sitemap and departs a second time. The ceilings exist for the truncated-but-valid
+ * child sitemap, which presents every URL it dropped as departed; a deployment that would rather
+ * lose that guard than lose departures sets them to -1 (`departureLimit`).
  */
 
 // Re-exported so callers reason about departures through one module rather than reaching into the
 // route compiler for the enum. It is DEFINED in routeClass.js because that is what validates the
 // route field, and one definition is what keeps the validator and the consumers spelling it alike.
 export { DepartureAction };
 
+/**
+ * A departure ceiling as configured, turned into the number the comparisons use.
+ *
+ * `sitemap.departure.maxActions` and `maxCandidates` spell "no ceiling" as -1 (see the schema for why
+ * not `null` or `Infinity`), and that has to become Infinity before it meets `acted >= limit` or
+ * `length < cap`: compared raw, -1 would refuse EVERY candidate — the exact opposite of what it says.
+ * Any negative reads as uncapped, since the schema's floor of -1 lets a fraction through. 0 stays 0,
+ * because 0 is "none" and is also how a walk with no opted-in route keeps collection off.
+ */
+export const departureLimit = (value) => (value < 0 ? Infinity : value);
+
+/**
+ * How many departed URLs one walk may hold for the post-walk check.
+ *
+ * Zero unless the check is on AND some route opts in, which keeps the candidate list empty — and the
+ * walk's `addRemoved` allocation-free — for every deployment that has not asked for this, whatever
+ * `maxCandidates` says. Resolved once per walk rather than per departed URL.
+ */
+export const departureCandidateCap = () =>
+	config.sitemap.departure.enabled && anyRouteDeparts() ? departureLimit(config.sitemap.departure.maxCandidates) : 0;
+
 /**
  * The action a route declares for URLs that leave a sitemap.
  *

packages/plugin/src/util/sitemapRun.js

@@ -126,7 +126,10 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar
 	// paginated index is pruned before the child that now claims it is reached — so mid-walk it is
 	// indistinguishable from one that genuinely left. Bounded: this is a list of URLs held in
 	// memory across a walk that can prune millions, and `departureCap: 0` (the default) keeps it
-	// empty for every deployment that has not opted a route in.
+	// empty for every deployment that has not opted a route in. `departureCap: Infinity` is the
+	// uncapped setting (`maxCandidates: -1`, via `departureLimit`), which holds every URL the walk
+	// unlinks and never sets `capped`. A URL past a finite cap is not deferred: it is already
+	// unlinked, so no later walk will offer it again — which is why `capped` is reported at all.
 	const departure = { candidates: [], capped: false, outcomes: {} };
 
 	return {