diff --git a/package-lock.json b/package-lock.json index 4d257f3c..e74b118c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.86.0", + "version": "0.87.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 932a121a..8259b07d 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -131,7 +131,7 @@ reasoning behind it. | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | | `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, `stored-unshared`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored), `vary-device` (only under `render.raw.deviceIndependent`: the origin's `Vary` named `User-Agent`, a client hint, the ingress device header, or `*`, declaring the document adaptive — any count means that option is wrong for this origin). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert. Under `render.raw.assumeShared` those documents are STORED and counted as `stored-unshared`; read `stored + stored-unshared` as the store rate. **That series is a census, not an alarm** — on an origin that sets a cookie on every response it is pinned at 100% from the first minute and cannot rise, so no threshold on it detects anything. What still detects personalization is re-running the body diff the option documents). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, `stored-unshared`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored), `vary-device` (only under `render.raw.deviceIndependent`: the origin's `Vary` named `User-Agent`, a client hint, the ingress device header, or `*`, declaring the document adaptive — any count means that option is wrong for this origin). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert. Under `render.raw.assumeShared` those documents are STORED and counted as `stored-unshared`; read `stored + stored-unshared` as the store rate. **That series is a census, not an alarm** — on an origin that sets a cookie on every response it is pinned at 100% from the first minute and cannot rise, so no threshold on it detects anything. What still detects personalization is re-running the body diff the option documents). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | | Metric | Kind | `path` | `method` | `type` | What it's for | | ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -143,7 +143,7 @@ reasoning behind it. | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | | `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | | Metric | Kind | `path` | `method` | `type` | What it's for | | ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -155,7 +155,7 @@ reasoning behind it. | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | | `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 8847e5ac..d6e6019e 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.86.0", + "version": "0.87.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index e8d98d87..41b91c4c 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -2082,23 +2082,47 @@ export const configSchema = group('Prerender plugin configuration.', { 'looked, and one that decides whether `maxActions` is a ceiling or a no-op. The result ' + 'and the progress row carry the outcome tally either way.' ), + // -1 is "no ceiling" for both caps below, and it is -1 rather than `null` or `Infinity` on + // purpose. `null` already means "no value" in every layer — the merge skips it and keeps the + // default, an override write refuses it, and the console's number field reports it for an + // emptied box — so making it mean "uncapped" would turn a blank `maxActions:` in config.yaml + // from the default into no guard at all, silently and in the dangerous direction. `Infinity` + // cannot cross JSON, so no override row or console edit could ever carry it. -1 is a plain + // number every layer already validates and transports, and nobody types it by accident. + // 0 keeps its meaning — act on / collect nothing — because 0 is also how a walk with no + // opted-in route switches collection off. Read them through `departureLimit`. maxActions: option( 5000, - 'Ceiling on departed URLs ACTED ON in one walk. Skipped candidates (re-attached, ' + - 'suppressed, route opted out) do not count against it. The cap is the guard against a ' + - 'pathological walk: a child sitemap that fetches truncated but still parses as valid XML ' + - 'presents every URL it no longer lists as departed, and without a ceiling one bad fetch ' + - 'would expire a large slice of the cache. Overflow is counted as `capped` and left for ' + - 'the next walk rather than silently dropped.', - { min: 0 } + 'Ceiling on departed URLs ACTED ON in one walk, or -1 for no ceiling. Skipped candidates ' + + '(re-attached, suppressed, route opted out) do not count against it.\n\n' + + 'WHAT IT GUARDS AGAINST: a child sitemap that fetches truncated but still parses as valid ' + + 'XML presents every URL it no longer lists as departed, and without a ceiling that one bad ' + + 'fetch would expire a large slice of the cache.\n\n' + + 'OVERFLOW IS DROPPED FOR GOOD, not deferred. By the time the check runs the walk has already ' + + 'unlinked the URL from its sitemap, and a later walk only looks for departures among URLs ' + + 'still linked to one — so a URL counted as `capped` is never offered again (unless it ' + + 're-enters a sitemap and departs a second time). It keeps its target, its render cadence ' + + 'and its cached page; it just never gets its route’s departure action. Read `capped` as ' + + 'departures lost, not postponed.\n\n' + + '-1 removes the ceiling: every departed URL gets its action and nothing is ever counted as ' + + '`capped`. That gives up the guard above, so read a `dryRun` tally first. 0 acts on ' + + 'nothing — every candidate is still decided and reported.', + { min: -1 } ), maxCandidates: option( 50000, - 'Ceiling on departed URLs HELD for the post-walk check. Separate from `maxActions` because ' + - 'this one bounds memory: it is a list of URLs retained across a walk that can prune ' + - 'millions. A capped list is reported as such, so a short list is never presented as ' + - '"few departed".', - { min: 0 } + 'Ceiling on departed URLs HELD for the post-walk check, or -1 for no ceiling. Separate from ' + + '`maxActions` because this one bounds memory: it is a list of URLs retained across a walk ' + + 'that can prune millions. Uncapped, the list is bounded by the URLs that walk actually ' + + 'unlinks — at most `scan.collectCap` per child sitemap — so it grows with how much of the ' + + 'corpus departs at once, which is exactly what a truncated child sitemap inflates. A capped ' + + 'list is reported as such (`departures.capped` on the result and the progress row), so a ' + + 'short list is never presented as "few departed".\n\n' + + 'Overflow is DROPPED FOR GOOD, for the same reason as `maxActions` overflow: the URL is ' + + 'already unlinked, so no later walk offers it again. It is never decided and never appears ' + + 'in the outcome tally; `removed` minus `departures.considered` is how many were lost. -1 ' + + 'removes the ceiling and never reports `capped`; 0 collects nothing.', + { min: -1 } ), } ), diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js index dc0eecd5..f97a650b 100644 --- a/packages/plugin/src/resources/Sitemap.js +++ b/packages/plugin/src/resources/Sitemap.js @@ -2,7 +2,7 @@ import { config, onConfigApplied } from '../config.js'; import { metrics } from '../metrics.js'; import { describeError } from '../util/errors.js'; import { Target } from './Target.js'; -import { anyRouteDeparts, classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js'; +import { classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js'; import { currentMinuteMs, epochMsOf, getInitialRenderTime, getNextSitemapRefreshTime } from '../util/time.js'; import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js'; import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js'; @@ -10,7 +10,7 @@ import { configuredStagingIp, dispatcherFor } from '../util/upstream.js'; import { setImmediate } from 'node:timers/promises'; import { applyInBatches, collectFromScan } from '../util/scan.js'; import { conditionalValidatorFor } from '../util/sitemapConditional.js'; -import { decideDeparture, DepartureAction } from '../util/sitemapDeparture.js'; +import { decideDeparture, DepartureAction, departureCandidateCap, departureLimit } from '../util/sitemapDeparture.js'; import { cacheKeysOf } from './Target.js'; import { writeSchedule } from '../util/renderSchedule.js'; import { resolveEffectiveInterval } from '../util/routeClass.js'; @@ -58,10 +58,9 @@ class Sitemap extends databases.sitemaps.Sitemap { const run = createRefreshRun({ removedSampleCap: config.sitemap.removedSampleCap, failedCap: config.sitemap.failedCap, - // Zero unless the departure check is on AND some route opts in, which keeps the candidate - // list empty — and `addRemoved` allocation-free — for every deployment that has not asked - // for this. `anyRouteDeparts` is resolved once per walk rather than per departed URL. - departureCap: config.sitemap.departure.enabled && anyRouteDeparts() ? config.sitemap.departure.maxCandidates : 0, + // Zero unless the departure check is on AND some route opts in; Infinity when + // `maxCandidates` is -1. See util/sitemapDeparture.js. + departureCap: departureCandidateCap(), }); const visited = new Set(); const queue = [{ url: rootSitemapUrl, parentUrl: null }]; @@ -750,12 +749,19 @@ function getTtlFromChangeFreq(changefreq, { minTtl, defaultTtl }) { * Every candidate is decided and counted, including the ones nothing happens to, so a dry run * answers the question a deployment actually has before enabling this: how many URLs a real walk * would touch, and how many of the departures are shear rather than departure. + * + * A candidate refused by `maxActions` is counted `capped` and is gone for good — the walk already + * unlinked it, so no later walk offers it again. `maxActions: -1` removes the ceiling. + * + * Exported for tests. */ -async function processDepartures(run) { +export async function processDepartures(run) { const urls = run.departureCandidates(); if (!urls.length) return; - const { dryRun, maxActions } = config.sitemap.departure; + const { dryRun } = config.sitemap.departure; + // Through `departureLimit`, never raw: -1 is "no ceiling", and `acted >= -1` would refuse all. + const maxActions = departureLimit(config.sitemap.departure.maxActions); let acted = 0; await applyInBatches({ @@ -827,7 +833,8 @@ async function processDepartures(run) { .map(([name, count]) => `${name} ${count}`) .join(', '); logger.info( - `[prerender] Departure check: ${considered} candidates${capped ? ' (CAPPED — some departed URLs were never checked)' : ''}` + + `[prerender] Departure check: ${considered} candidates` + + `${capped ? ' (CAPPED at maxCandidates — the departed URLs past it were never checked, and no later walk will offer them)' : ''}` + `${dryRun ? ', DRY RUN' : ''} — ${summary || 'nothing to do'}` ); diff --git a/packages/plugin/src/util/sitemapDeparture.js b/packages/plugin/src/util/sitemapDeparture.js index 0b643815..c69b2486 100644 --- a/packages/plugin/src/util/sitemapDeparture.js +++ b/packages/plugin/src/util/sitemapDeparture.js @@ -1,5 +1,5 @@ import { config } from '../config.js'; -import { classifyUrl, DepartureAction, PRERENDER } from './routeClass.js'; +import { anyRouteDeparts, classifyUrl, DepartureAction, PRERENDER } from './routeClass.js'; /** * What a URL LEAVING a sitemap means, and what to do about it. @@ -39,6 +39,16 @@ import { classifyUrl, DepartureAction, PRERENDER } from './routeClass.js'; * Candidates are collected during the walk and re-read once it has finished; anything that picked * up an attribution in the meantime is dropped. That is also why this cannot be folded into * `reconcileSitemapEntries`. + * + * ── WHY A CAPPED DEPARTURE IS GONE FOR GOOD ────────────────────────────────────────────────── + * + * The prune unlinks a departed target (`sitemapUrl -> null`) BEFORE it becomes a candidate, and a + * later walk finds departures by scanning for targets still linked to the sitemap being pruned. So + * a URL that overflows either ceiling — past `maxCandidates` it is never held, past `maxActions` it + * is decided but not acted on — is not left for the next walk: no walk will offer it again unless + * it re-enters a sitemap and departs a second time. The ceilings exist for the truncated-but-valid + * child sitemap, which presents every URL it dropped as departed; a deployment that would rather + * lose that guard than lose departures sets them to -1 (`departureLimit`). */ // Re-exported so callers reason about departures through one module rather than reaching into the @@ -46,6 +56,27 @@ import { classifyUrl, DepartureAction, PRERENDER } from './routeClass.js'; // route field, and one definition is what keeps the validator and the consumers spelling it alike. export { DepartureAction }; +/** + * A departure ceiling as configured, turned into the number the comparisons use. + * + * `sitemap.departure.maxActions` and `maxCandidates` spell "no ceiling" as -1 (see the schema for why + * not `null` or `Infinity`), and that has to become Infinity before it meets `acted >= limit` or + * `length < cap`: compared raw, -1 would refuse EVERY candidate — the exact opposite of what it says. + * Any negative reads as uncapped, since the schema's floor of -1 lets a fraction through. 0 stays 0, + * because 0 is "none" and is also how a walk with no opted-in route keeps collection off. + */ +export const departureLimit = (value) => (value < 0 ? Infinity : value); + +/** + * How many departed URLs one walk may hold for the post-walk check. + * + * Zero unless the check is on AND some route opts in, which keeps the candidate list empty — and the + * walk's `addRemoved` allocation-free — for every deployment that has not asked for this, whatever + * `maxCandidates` says. Resolved once per walk rather than per departed URL. + */ +export const departureCandidateCap = () => + config.sitemap.departure.enabled && anyRouteDeparts() ? departureLimit(config.sitemap.departure.maxCandidates) : 0; + /** * The action a route declares for URLs that leave a sitemap. * diff --git a/packages/plugin/src/util/sitemapRun.js b/packages/plugin/src/util/sitemapRun.js index d6d603f7..f2700bf4 100644 --- a/packages/plugin/src/util/sitemapRun.js +++ b/packages/plugin/src/util/sitemapRun.js @@ -126,7 +126,10 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar // paginated index is pruned before the child that now claims it is reached — so mid-walk it is // indistinguishable from one that genuinely left. Bounded: this is a list of URLs held in // memory across a walk that can prune millions, and `departureCap: 0` (the default) keeps it - // empty for every deployment that has not opted a route in. + // empty for every deployment that has not opted a route in. `departureCap: Infinity` is the + // uncapped setting (`maxCandidates: -1`, via `departureLimit`), which holds every URL the walk + // unlinks and never sets `capped`. A URL past a finite cap is not deferred: it is already + // unlinked, so no later walk will offer it again — which is why `capped` is reported at all. const departure = { candidates: [], capped: false, outcomes: {} }; return { diff --git a/packages/plugin/test/sitemapDepartureCap.test.js b/packages/plugin/test/sitemapDepartureCap.test.js new file mode 100644 index 00000000..3cc1226d --- /dev/null +++ b/packages/plugin/test/sitemapDepartureCap.test.js @@ -0,0 +1,246 @@ +/** + * The two departure ceilings, `sitemap.departure.maxActions` and `maxCandidates`, and their + * uncapped setting. + * + * -1 means "no ceiling" for both. It has to be honoured by every layer that can set it (config.yaml, + * a stored override) and by both places that enforce it (the walk's candidate list and the post-walk + * action gate), and it must never report `capped` — a URL past either ceiling is dropped for good, + * because the walk has already unlinked it, so "capped" is a statement about lost departures and an + * uncapped deployment must never make it. + * + * `processDepartures` runs for real here against fake tables: the gate is the code under test, and a + * copy of its comparison in a helper test would pass whatever the resource does. + */ +import { test, before } from 'node:test'; +import assert from 'node:assert/strict'; + +let applyOptions, config, resolveConfig, defaultConfig; +let validateOverride; +let createRefreshRun; +let departureCandidateCap, departureLimit; +let processDepartures; + +const warns = []; +// Every URL the post-walk re-read asks about, and what it answers: unlinked, not suppressed — a +// genuine departure on an opted-in route, so each one reaches the action gate. +const reads = []; +const pageReads = []; + +before(async () => { + globalThis.Resource = class {}; + globalThis.server = { + hostname: 'node-1', + workerIndex: 0, + nodes: [], + config: { http: { port: 9926 } }, + recordAnalytics() {}, + }; + globalThis.logger = { debug() {}, info() {}, warn: (msg) => warns.push(String(msg)), error() {} }; + // Sitemap.js's import graph destructures Harper tables at module load (see + // test/configReactivity.test.js for why each of these is needed). + const sabs = new Map(); + globalThis.databases = { + render_service: { + Target: class { + static async get({ id }) { + reads.push(id); + return { url: id, sitemapUrl: null, state: null, renderInterval: null, demandInterval: null }; + } + }, + QueueControl: class {}, + }, + render_schedule: { RenderSchedule: class {} }, + page_cache: { + PrerenderedPage: class { + // No cached page, so the live `expire` path reads and writes nothing further — enough to + // prove the action ran without faking the page store. + static async get({ id }) { + pageReads.push(id); + return null; + } + }, + }, + sitemaps: { Sitemap: class {}, SitemapRefresh: class {} }, + coordination: { + SharedBuffer: class { + static primaryStore = { + getUserSharedBuffer: (key, buffer) => { + if (!sabs.has(key)) sabs.set(key, buffer); + return sabs.get(key); + }, + tryLock: () => true, + unlock() {}, + }; + }, + }, + }; + + ({ applyOptions, config, resolveConfig } = await import('../src/config.js')); + ({ defaultConfig } = await import('../src/configSchema.js')); + ({ validateOverride } = await import('../src/util/configOverride.js')); + ({ createRefreshRun } = await import('../src/util/sitemapRun.js')); + ({ departureCandidateCap, departureLimit } = await import('../src/util/sitemapDeparture.js')); + ({ processDepartures } = await import('../src/resources/Sitemap.js')); +}); + +const PRODUCT_ROUTE = { match: 'prefix', path: '/product/', mode: 'prerender' }; + +const configure = ({ departure = {}, action = 'render' } = {}) => { + warns.length = 0; + return applyOptions({ + sitemap: { departure: { enabled: true, dryRun: true, ...departure } }, + ingress: { + mode: 'forwarded', + routes: [{ ...PRODUCT_ROUTE, ...(action ? { departureAction: action } : {}) }], + }, + }); +}; + +const urls = (n) => Array.from({ length: n }, (_, i) => ({ url: `https://site.example.com/product/prd-${i}` })); + +/** A walk that unlinked `n` product URLs, collected under the configured candidate cap. */ +const walkThatUnlinked = (n) => { + const run = createRefreshRun({ departureCap: departureCandidateCap() }); + run.addRemoved(urls(n)); + return run; +}; + +// ---- the defaults are not moved by this ---- + +test('the defaults are unchanged: 5000 actions, 50000 candidates', () => { + const { departure } = defaultConfig().sitemap; + assert.equal(departure.maxActions, 5000); + assert.equal(departure.maxCandidates, 50000); +}); + +test('departureLimit: -1 is Infinity, 0 stays 0, a ceiling stays itself', () => { + assert.equal(departureLimit(-1), Infinity); + assert.equal(departureLimit(0), 0); + assert.equal(departureLimit(5000), 5000); +}); + +// ---- both layers can set it ---- + +test('config.yaml can set -1 on both, and it is kept rather than reset to the default', () => { + configure({ departure: { maxActions: -1, maxCandidates: -1 } }); + assert.equal(config.sitemap.departure.maxActions, -1); + assert.equal(config.sitemap.departure.maxCandidates, -1); + assert.deepEqual( + warns.filter((w) => w.includes('sitemap.departure')), + [], + '-1 is a value, not a rejection' + ); +}); + +test('below -1 is still rejected, keeping the default', () => { + configure({ departure: { maxActions: -2, maxCandidates: -5 } }); + assert.equal(config.sitemap.departure.maxActions, 5000); + assert.equal(config.sitemap.departure.maxCandidates, 50000); + assert.ok(warns.some((w) => w.includes('prerender.sitemap.departure.maxActions') && w.includes('>= -1'))); +}); + +test('a config-override row accepts -1 at the door and applies it', () => { + for (const path of ['sitemap.departure.maxActions', 'sitemap.departure.maxCandidates']) { + assert.equal(validateOverride(path, -1).ok, true, `${path}: -1 must be writable from the console`); + assert.equal(validateOverride(path, -2).ok, false, `${path}: below -1 must be refused`); + assert.equal(validateOverride(path, 0).ok, true, `${path}: 0 is still a valid setting`); + } + const { config: resolved } = resolveConfig( + {}, + { 'sitemap.departure.maxActions': -1, 'sitemap.departure.maxCandidates': -1 } + ); + assert.equal(resolved.sitemap.departure.maxActions, -1); + assert.equal(resolved.sitemap.departure.maxCandidates, -1); +}); + +test('an override of -1 wins over a finite config.yaml ceiling', () => { + const { config: resolved } = resolveConfig( + { sitemap: { departure: { maxActions: 100 } } }, + { 'sitemap.departure.maxActions': -1 } + ); + assert.equal(resolved.sitemap.departure.maxActions, -1); +}); + +// ---- the candidate list ---- + +test('maxCandidates -1 collects every departed URL and never reports capped', () => { + configure({ departure: { maxCandidates: -1 } }); + assert.equal(departureCandidateCap(), Infinity); + + // Past the default ceiling, so an uncapped setting that quietly fell back to it would show. + const run = walkThatUnlinked(60000); + const { departures, removed } = run.snapshot(); + assert.equal(departures.considered, 60000); + assert.equal(departures.considered, removed, 'none dropped: every unlinked URL is a candidate'); + assert.equal(departures.capped, false); +}); + +test('the default candidate ceiling still caps, and says so', () => { + configure(); + const run = walkThatUnlinked(60000); + assert.equal(run.snapshot().departures.considered, 50000); + assert.equal(run.snapshot().departures.capped, true); +}); + +test('maxCandidates 0 still collects nothing, and is not reported as overflow', () => { + configure({ departure: { maxCandidates: 0 } }); + assert.equal(departureCandidateCap(), 0); + const run = walkThatUnlinked(10); + assert.equal(run.snapshot().departures.considered, 0); + assert.equal(run.snapshot().departures.capped, false); +}); + +// -1 must not become a way to switch collection ON: 0 is how a walk with nothing opted in stays +// allocation-free, whatever the ceiling says. +test('uncapped collects nothing when no route opts in or the check is off', () => { + configure({ departure: { maxCandidates: -1 }, action: null }); + assert.equal(departureCandidateCap(), 0, 'no route opts in'); + configure({ departure: { maxCandidates: -1, enabled: false } }); + assert.equal(departureCandidateCap(), 0, 'master switch off'); +}); + +// ---- the action gate ---- + +test('maxActions -1 acts on every candidate and never counts capped (dry run)', async () => { + configure({ departure: { maxActions: -1, maxCandidates: -1 } }); + const run = walkThatUnlinked(6000); + await processDepartures(run); + + const { outcomes } = run.snapshot().departures; + assert.equal(outcomes['would-render'], 6000, 'past the default 5000, every one is acted on'); + assert.equal(outcomes.capped, undefined); +}); + +test('maxActions -1 acts on every candidate for real, not only in a dry run', async () => { + configure({ departure: { maxActions: -1, maxCandidates: -1, dryRun: false }, action: 'expire' }); + pageReads.length = 0; + const run = walkThatUnlinked(5200); + await processDepartures(run); + + const { outcomes } = run.snapshot().departures; + assert.equal(outcomes.expire, 5200); + assert.equal(outcomes.capped, undefined); + assert.ok(pageReads.length >= 5200, 'every departed URL had its cached pages looked up to expire'); +}); + +test('the default maxActions still caps at 5000 and counts the rest as capped', async () => { + configure(); + const run = walkThatUnlinked(6000); + await processDepartures(run); + + const { outcomes } = run.snapshot().departures; + assert.equal(outcomes['would-render'], 5000); + assert.equal(outcomes.capped, 1000); +}); + +test('maxActions 0 still acts on nothing: every candidate is decided and counted capped', async () => { + configure({ departure: { maxActions: 0 } }); + reads.length = 0; + const run = walkThatUnlinked(25); + await processDepartures(run); + + const { outcomes } = run.snapshot().departures; + assert.equal(outcomes['would-render'], undefined); + assert.equal(outcomes.capped, 25); + assert.equal(reads.length, 25, 'each candidate is still re-read and decided'); +});