diff --git a/package-lock.json b/package-lock.json index 1f6fcd49..b6dc9060 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.87.0", + "version": "0.88.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 8259b07d..8416b397 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,42 +121,42 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ------------------ | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, `stored-unshared`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored), `vary-device` (only under `render.raw.deviceIndependent`: the origin's `Vary` named `User-Agent`, a client hint, the ingress device header, or `*`, declaring the document adaptive — any count means that option is wrong for this origin). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert. Under `render.raw.assumeShared` those documents are STORED and counted as `stored-unshared`; read `stored + stored-unshared` as the store rate. **That series is a census, not an alarm** — on an origin that sets a cookie on every response it is pinned at 100% from the first minute and cannot rise, so no threshold on it detects anything. What still detects personalization is re-running the body diff the option documents). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ------------------ | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders; from v0.88.0 it also counts pages wrong on an ARMED `pageCheck.fields` mapping — the per-field split is `fieldMismatch` in the pass record, deliberately not a series), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, `stored-unshared`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored), `vary-device` (only under `render.raw.deviceIndependent`: the origin's `Vary` named `User-Agent`, a client hint, the ingress device header, or `*`, declaring the document adaptive — any count means that option is wrong for this origin). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert. Under `render.raw.assumeShared` those documents are STORED and counted as `stored-unshared`; read `stored + stored-unshared` as the store rate. **That series is a census, not an alarm** — on an origin that sets a cookie on every response it is pinned at 100% from the first minute and cannot rise, so no threshold on it detects anything. What still detects personalization is re-running the body diff the option documents). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders; from v0.88.0 it also counts pages wrong on an ARMED `pageCheck.fields` mapping — the per-field split is `fieldMismatch` in the pass record, deliberately not a series), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per governed DEVICE VARIANT — a two-device job emits two — so shares read against rendered devices (`render.time_ms`), not posted results) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that PAGE (URL + device) last produced — silent on a first render and after a rebaseline, and never a comparison across devices. `rebaseline` (counter) is a page whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per variant. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` = departed URLs `maxActions` refused, DROPPED FOR GOOD (-1 removes the cap)), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders; from v0.88.0 it also counts pages wrong on an ARMED `pageCheck.fields` mapping — the per-field split is `fieldMismatch` in the pass record, deliberately not a series), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: @@ -307,7 +307,7 @@ console UI consuming these routes is the separate `@harperfast/prerender-console | `POST explain` | **Per-URL diagnosis**: cache key, route match, freshness verdict, schedule row, residency owner, suppression state. The first call to make about one URL; the metrics can't name a URL by construction. | | `POST queue` | Queue pause: desired (replicated intent) vs. observed per node. | | `POST backlog` / `POST reconcile` | Force a snapshot / a schedule-gap repair sweep. Both are **scans** — operator actions, not dashboard polls. | -| `GET change-probe` | The change probe's node-local state: compiled rules, the last sweep/canary pass records (with failure samples the `probe_*` counters cannot carry), canary cohort sizes, and armed intervals. `POST change-probe` forces a pass — a **paced origin-fetch sweep**, an operator action, never a dashboard poll. | +| `GET change-probe` | The change probe's node-local state: compiled rules, the last sweep/canary pass records (with failure samples the `probe_*` counters cannot carry — and, from v0.88.0, `slotChanges` (origin changes per extract slot), `fieldMismatch` (page mismatches per mapped field), `caughtUp`/`ignored` (changes absorbed without a render) and `fieldGuard` (the mapping-defect guard's recent witnessed counts and which fields are disarmed)), canary cohort sizes, and armed intervals. `POST change-probe` forces a pass — a **paced origin-fetch sweep**, an operator action, never a dashboard poll. | ### 4b. Log lines that carry numbers no metric has @@ -321,6 +321,14 @@ Grep-able, `[prerender]`-prefixed, and each is the richer record of something th "target with no schedule row" state. Any non-zero `restored` is worth an alert; nothing else reports it. - **`Sitemap refresh for finished: …`** — sitemaps processed, targets created/removed, duration. +- **`change-probe sweep {…}` / `change-probe canary {…}`** — the whole pass record, including the + decompositions no series carries: `slotChanges` (which extract slot moved), `fieldMismatch` (which + mapped page field disagreed), `caughtUp` and `ignored`. Read `caughtUp / changed` as the renders the + page record saved; a `fieldMismatch` entry that tracks the matched count pass after pass is a broken + mapping (the guard below should already have disarmed it). +- **`change-probe : pageCheck field : … is DISARMED`** — the mapping-defect guard + (`changeProbe.mappingGuard`) found a mapped field disagreeing with most witnessed pages and stopped it + triggering. Nothing else announces it; fix the mapping (an edit re-arms it) or remove it. - **Render verdicts per URL** — `Suppressing prerendered url …`, `Prerender failed for … (reason)`, `Retrying … (failure strike N)`, `redirected to … which is `. The aggregate is the `render` metric's `outcome` series; these lines carry the URL. Most are `info`/`debug` now (they are normal diff --git a/packages/plugin/package.json b/packages/plugin/package.json index d6e6019e..548c18a3 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.87.0", + "version": "0.88.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 41b91c4c..66bbc89b 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -934,9 +934,12 @@ export const configSchema = group('Prerender plugin configuration.', { '"payload.products[0].prices[0].salePrice" — the extracted values ARE the watched content; ' + 'everything else in the response is ignored. A path may end at an object or array, which is ' + 'signed whole, and `[*]` projects the rest of the path over every element of an array ' + - '("payload.products[0].variants[*].availability" -> one value per variant, in the endpoint’s ' + - 'order) — the way to watch per-variant state without signing fields that move on their own ' + - '(inventory counters, store data). An extraction where every path yields null is a ' + + '("payload.products[0].variants[*].availability" -> one value per variant) — the way to watch ' + + 'per-variant state without signing fields that move on their own ' + + '(inventory counters, store data). A trailing tuple, `[*].{a,b.c}`, projects each element to ' + + '`[a, b.c]` (dotted inner paths, no brackets) so per-variant fields stay attached to their key — ' + + 'e.g. "payload.products[0].variants[*].{sku,availability,price.value}" for a `skus` page check. ' + + 'Projections (tuples included) are sorted, so a reordered array is not a change. An extraction where every path yields null is a ' + 'FAILED probe, never a new signature, so an endpoint shape change cannot mass-trigger.\n' + ' EDITING A RULE. The rule’s observation (endpoint, method, headers, body, extract, ' + 'statusSignals) is fingerprinted and stored beside every baseline. Change any of it and each ' + @@ -995,6 +998,47 @@ export const configSchema = group('Prerender plugin configuration.', { 'with its own vocabulary. A per-variant list reads in stock when ANY variant is, exactly as the ' + 'page’s offers are read. ' + 'Detection is one extra node-local write per render and no extra origin traffic.\n' + + '\n\nTHE PAGE RECORD (`pageCheck.fields`). The claim pair above generalizes to every field a page ' + + 'visibly states. With @harperfast/prerender-browser >= 1.37.0 each render also posts `pageFacts` ' + + '(canonical, title, metaDescription, h1, product.name, product.brand, product.image, ' + + 'product.rating.value, product.rating.count, product.offers, breadcrumbs), and a rule that maps ' + + 'fields stores them per URL and compares each mapped field on every probe. That does three things: ' + + '(1) a page that DISAGREES with the origin on any mapped field is re-rendered, like a claim-pair ' + + 'disagreement; (2) an origin CHANGE that the cached page already shows (a cadence render landed after ' + + 'it) is NOT re-rendered — every changed slot must be a mapped slot whose record agrees with the new ' + + 'value, and the baseline simply moves (counted `caughtUp`); (3) with `ignoreChanges`, changes to ' + + 'fields the page cannot show stop re-rendering at all. Shape: `pageCheck: { enabled: true, ' + + 'fields: [{ slot: , fact: "", compare: "" }, ...], ignoreChanges: [, ...] }` ' + + '(both, like the claim pair, apply only while `enabled` is true), where `slot` indexes ' + + '`extract`. The comparators are a closed set, each applying to certain facts: ' + + '"text" (exact after Unicode NFC, whitespace collapse and trim — no case folding and NO HTML ' + + 'stripping: an endpoint’s description may carry markup the page’s meta tag repeats verbatim), ' + + '"path" (URL path only — origin, query and fragment ignored, a relative value resolved against the ' + + 'page URL; for canonical and product.image, whose URLs typically differ by size parameters alone), ' + + '"number" (numeric equality, "4.0" = 4; optional `tolerance`), "priceSet" (against the SET of prices ' + + 'the page’s offers print: a single endpoint price must be among them, and for a list EVERY price ' + + 'the page prints must still be in the endpoint’s set — not equality, so a page that lists only some ' + + 'variants cannot disagree forever; use "skus" for per-variant exactness), "names" (an ordered list of ' + + 'strings, or of objects carrying `nameKey`, default "name", matched against the TAIL of the page’s ' + + 'breadcrumbs — a leading home crumb needs no configuration; extract the list itself, not a `[*]` ' + + 'projection, since projections are sorted) and "skus" (per-variant tuples from a `[*].{…}` projection ' + + 'against the page’s offers keyed by SKU, over the SKUs both list; `tuple` names each tuple ' + + 'position, default ["sku", "availability", "price"]). EVERY COMPARATOR FAILS TOWARD NO CLAIM: null, ' + + 'absent or unparseable on either side is not compared, never a disagreement, and never evidence ' + + 'that a page caught up. A bad entry is dropped alone with a warning. A mapping that is wrong anyway ' + + 'is caught by `changeProbe.mappingGuard`. The record is stored only for a rule that maps fields, ' + + 'from the first device variant, and only when the render stored every default device (otherwise ' + + 'null — a partial render cannot vouch for the device page it did not replace); records over 16 KB ' + + 'are refused, not truncated. `pageCheck` may carry `fields` and/or `ignoreChanges` without ' + + 'priceFrom/availableFrom. Like the rest of pageCheck, none of this is in the rule fingerprint: ' + + 'adding or editing a mapping re-baselines nothing.\n' + + ' pageCheck.ignoreChanges extract indices whose origin changes never trigger a re-render — ' + + 'fields the page cannot show (an inventory counter the endpoint returns beside the price). A change ' + + 'confined to them writes the new baseline and triggers nothing (counted `ignored`), and the canary ' + + 'does not count it as a change, so a site-wide edit of such a field cannot trip a bulk ' + + 'invalidation. A status-signal literal is a state, not slots, and is never ignored.\n' + + ' PER-SLOT STATS. Pass records decompose `changed` by slot (`slotChanges`) and page mismatches by ' + + 'mapped field (`fieldMismatch`), in `GET /prerender_admin/change-probe` and the pass log line.\n' + ' invalidateScope optional invalidation scope ("all" or "route::") the canary ' + 'records on a mass change. Empty = the canary detects and logs only.\n' + ' label optional name for logs and the admin surface.', @@ -1295,6 +1339,46 @@ export const configSchema = group('Prerender plugin configuration.', { ), } ), + mappingGuard: group( + 'The MAPPING-DEFECT GUARD for `pageCheck.fields`. A field mapped to the wrong slot, or compared the ' + + 'wrong way, disagrees with nearly every page — and every disagreement is a re-render, so one bad ' + + 'mapping would otherwise re-render its whole corpus on every pass. A correct mapping disagrees only ' + + 'when the page really is wrong, and on a WITNESSED page (rendered after the stored baseline was ' + + 'taken, the origin unchanged since) that takes a genuine round trip — the value changed and changed ' + + 'back between two probes with a render in between, measured well under 1%.\n\n' + + 'So each node counts, per mapped field, witnessed comparisons and witnessed disagreements, and ' + + 'DISARMS the field when its disagreement rate reaches `threshold` over at least `minWitnessed` ' + + 'comparisons: it stops triggering re-renders (and stops vouching for caught-up changes and ' + + 'verifications), a warning names the rule and slot, and its mismatches are still counted ' + + '(`fieldMismatch` in the pass record; the guard’s own counts are `fieldGuard`). It stays disarmed ' + + 'until that field’s mapping is edited or the process restarts. It never suppresses an individual ' + + 'disagreement — a single one is exactly the round trip the page check exists to catch. Disarming ' + + 'never hides an ORIGIN change: those trigger on the signature as always; a disarmed field only stops ' + + 'correcting pages that are wrong while the origin is unchanged.\n\n' + + 'The rate is RECENT, not lifetime: both counts halve each time the sample reaches ten times ' + + '`minWitnessed`, so a mapping that was right for a week and then broke (the site changed its page ' + + 'template) is disarmed within a few thousand comparisons instead of after outweighing the week.\n\n' + + 'In dry run the guard counts and disarms too, which is the point of a dry-run week: a broken ' + + 'mapping announces itself before anything is armed.', + { + threshold: option( + 0.2, + 'Witnessed disagreement rate at or above which a mapped field is disarmed. The default sits ' + + 'far from both ends: a wrong mapping disagrees on nearly every comparison, a correct one on ' + + 'the rare round trip (a correct mapping would need twenty times the measured round-trip ' + + 'rate to reach it).', + { min: 0, max: 1 } + ), + minWitnessed: option( + 200, + 'Fewest witnessed comparisons of a field before `threshold` is consulted. Large enough that a ' + + 'correct mapping cannot reach the threshold by chance (at a 1% true rate, 20% of 200 is ~40 ' + + 'disagreements where ~2 are expected); small enough that a broken mapping is disarmed after ' + + 'about this many spurious re-renders per node rather than a corpus of them.', + { min: 1 } + ), + } + ), } ), diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js index 6b9912da..da52b833 100644 --- a/packages/plugin/src/resources/RenderQueue.js +++ b/packages/plugin/src/resources/RenderQueue.js @@ -827,6 +827,15 @@ export class RenderQueue extends Resource { // for its (node-local) write: a render must not fail because a probe optimisation could not // be recorded. const claiming = stored.find((variant) => variant.structuredOffers !== undefined) ?? stored[0]; + // The page record, chosen the same way from the variant that ran ITS extraction (browser + // >= 1.37.0). It is one record for the URL, so it only vouches for the URL when this result + // replaced EVERY default device's page: after a partial render (one device failed) or a + // one-device render, the other device's cached page is older than the record would claim, + // and a record that "caught up" with a change would then skip re-rendering a page that did not. + const describing = stored.find((variant) => variant.pageFacts !== undefined) ?? stored[0]; + const everyDevice = defaultDeviceTypes().every((device) => + stored.some((variant) => variant.deviceType === device) + ); // CONCURRENTLY, because these all sit inside ONE request-scoped transaction and its duration // is the thing to keep short. A result now carries every device of the URL, so where the // per-key path wrote one page blob per transaction this writes one per device, in series — @@ -850,7 +859,10 @@ export class RenderQueue extends Resource { // `recordPageClaim` it never rejects — a regression signal must not cost a render. const observed = stored.filter((variant) => hasObservations(variant.readiness?.learned)); await Promise.all([ - recordPageClaim(scheduleUrl, claiming.structuredOffers, cachedAt), + recordPageClaim(scheduleUrl, claiming.structuredOffers, cachedAt, { + pageFacts: describing.pageFacts, + complete: everyDevice, + }), ...observed.map((variant) => recordReadinessExpectation({ url: scheduleUrl, deviceType: variant.deviceType }, variant.readiness) ), diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index 39105a59..67ebc492 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -428,8 +428,23 @@ type ProbeState @table(database: "probe_state", replicate: false) @sealed { # recorded. Those differ by milliseconds, and the difference is load-bearing: this value is # copied onto a verification as the basis it certifies, and a stamp taken a few ms after the page # was written would make the very page it certifies fail its own `lastCached >= basisAt` test. - # Written in the SAME patch as `pageSignature`, so it costs no extra read and no extra write. + # Written in the SAME patch as `pageSignature`, so it costs no extra read and no extra write. It + # stamps `pageFacts` too, and the mapping-defect guard reads it: a record stamped AFTER the + # baseline's `probedAt`, compared against an unchanged origin, is a WITNESSED comparison. pageClaimAt: Date + # THE PAGE RECORD: what the cached page claims for every field it visibly states — canonical, + # title, meta description, h1, product name/brand/image/rating/offers, breadcrumbs — as canonical + # JSON (fixed key order), from the renderer's `pageFacts` (@harperfast/prerender-browser >= 1.37.0). + # Compared on every probe against the slots a rule maps (`pageCheck.fields`), so the probe + # re-renders a page that is WRONG on any mapped field and skips one that already caught up with an + # origin change. Written by the render path only, in the SAME patch as `pageSignature` and + # `pageClaimAt` (no extra read or write), and only for a rule that maps fields — always the whole + # record, so it never describes an older render than `pageClaimAt` says: null when the render's + # facts are unknown (older renderer, nothing extracted, not every default device stored) or the + # record exceeds 16 KB (refused, never truncated). Cleared with the claim when a trigger + # hard-expires the page. Nullable and inert unless a rule maps fields. ADDING THIS COLUMN to the + # @sealed table needs the component's workers restarted (a deploy does that), not a config reload. + pageFacts: String # The fingerprint of the RULE that produced `signature` (changeProbeSpec.js ruleFingerprint): # endpoint, method, headers, body, extract paths, status signals. A baseline is only comparable # to an observation made the same way, so a probe whose rule fingerprint differs from the stored diff --git a/packages/plugin/src/util/changeProbe.js b/packages/plugin/src/util/changeProbe.js index fc23a952..27a4d08a 100644 --- a/packages/plugin/src/util/changeProbe.js +++ b/packages/plugin/src/util/changeProbe.js @@ -75,6 +75,13 @@ import { apiClaimOf, claimsDisagree, pageClaimFromOffers, + changedSlots, + compareField, + fieldCaughtUp, + parsePageFacts, + serializePageFacts, + signatureSlots, + PAGE_FACTS_MAX_BYTES, } from './changeProbeSpec.js'; const targetTable = () => databases.render_service.Target; @@ -149,7 +156,11 @@ const newStats = () => ({ failed: 0, // fetch/parse/extraction failures — signature untouched, nothing triggered errors: 0, // trigger writes that threw fresh: 0, // skipped: baseline younger than reprobeAfter (a pass already covered it) - pageMismatch: 0, // cached page disagreed with the origin (pageCheck) — OVERLAYS the buckets above, which count by signature outcome alone + pageMismatch: 0, // cached page disagreed with the origin (pageCheck: the claim pair, or an ARMED mapped field) — OVERLAYS the buckets above, which count by signature outcome alone + caughtUp: 0, // origin changed, but every changed slot is a mapped slot the cached page ALREADY shows the new value for: baseline written, nothing triggered — OVERLAYS changed + ignored: 0, // origin changed only in pageCheck.ignoreChanges slots: baseline written, nothing triggered, NOT a change for the canary — OVERLAYS unchanged + slotChanges: {}, // { [rule]: { [slot]: n } } origin changes per extract slot ("signal" = a status-signal literal was involved, so no slot can be named) — decomposes changed, caughtUp and ignored + fieldMismatch: {}, // { [rule]: { [":"]: n } } page mismatches per mapped field, armed or not — a disarmed field still counts here, and only here throttled: 0, // probes the origin refused with a pushback status — what drives the backoff throttleLevel: 1, // ORIGIN pacing-window multiplier when the pass ended; 1 means never backed off loadThrottleLevel: 1, // LOCAL (event-loop) multiplier when the pass ended; 1 means never backed off @@ -343,7 +354,7 @@ export const triggerRevalidate = async (row) => { const readSignature = async (url) => { const row = await probeStateTable().get({ id: url, - select: ['url', 'signature', 'probedAt', 'pageSignature', 'pageClaimAt', 'ruleFingerprint'], + select: ['url', 'signature', 'probedAt', 'pageSignature', 'pageClaimAt', 'pageFacts', 'ruleFingerprint'], }); if (!row) return null; // A Date column can surface as a Date, an epoch number, a string, or — the trap this coercion @@ -358,6 +369,8 @@ const readSignature = async (url) => { probedAt, pageSignature: row.pageSignature ?? null, pageClaimAt: row.pageClaimAt ?? null, + // The page record (canonical JSON string), parsed lazily by processOne only for a rule that maps fields. + pageFacts: row.pageFacts ?? null, // null for a row written before fingerprints existed — read as "the rule in force", see processOne. fingerprint: row.ruleFingerprint ?? null, }; @@ -378,18 +391,22 @@ export const writeSignature = (url, signature, { rowExists = false, clearClaim = // can tell a rule edit from a content change. Written with EVERY baseline, including the // one-time stamp of a pre-fingerprint row; a signature is never left beside a stale fingerprint. if (fingerprint !== null) fields.ruleFingerprint = fingerprint; - // BOTH HALVES OF THE PAIR, always. `pageClaimAt` is the render `pageSignature` came from, and the - // two are only meaningful together — a claim with no basis cannot be scoped to a device page, and - // a basis with no claim describes nothing. Clearing one and leaving the other is inert today + // THE WHOLE RECORD, always. `pageClaimAt` is the render `pageSignature` and `pageFacts` came from, + // and they are only meaningful together — a claim with no basis cannot be scoped to a device page, + // and a basis with no claim describes nothing. Clearing one and leaving another is inert today // (the verification write requires `stored.pageSignature`), but it leaves a half-state a later - // reader can pick up, which is the shape of bug this module's own comments keep closing. + // reader can pick up, which is the shape of bug this module's own comments keep closing. The page + // record goes too: the trip hard-expired the page it described, so a surviving record would be + // compared against the origin as though that page were still served — and, for a record that + // disagrees, re-trigger the already-expired page on every pass until its re-render lands. if (clearClaim) { fields.pageSignature = null; fields.pageClaimAt = null; + fields.pageFacts = null; } return rowExists ? probeStateTable().patch(url, fields) - : probeStateTable().put(url, { url, pageSignature: null, pageClaimAt: null, ...fields }); + : probeStateTable().put(url, { url, pageSignature: null, pageClaimAt: null, pageFacts: null, ...fields }); }; /** @@ -398,8 +415,28 @@ export const writeSignature = (url, signature, { rowExists = false, clearClaim = * a render must never fail because a probe optimisation could not be recorded. Extraction that * yields nothing writes nothing — same rule as a failed probe, so a markup change cannot mass- * trigger by making every page look like a disagreement. + * + * TWO PARTS, ONE WRITE. The price/availability claim (`pageSignature`, from the renderer's + * `structuredOffers`) for a rule with the claim pair, and the PAGE RECORD (`pageFacts`, from the + * renderer's `pageFacts`) for a rule that maps fields. Both ride in the same patch as + * `pageClaimAt`, so the record costs no extra read and no extra write. + * + * A RULE THAT MAPS FIELDS ALWAYS WRITES THE WHOLE RECORD, nulls included. The record's contract is + * "what the page served right now claims", and `pageClaimAt` is the render it describes: leaving an + * older render's facts (or claim) in place beside a newer stamp would compare a page that is no + * longer served, and credit it to the wrong render. So a render whose facts are unknown — the + * renderer predates them (ABSENT), the extraction found nothing (null), the record is oversized, or + * the render did not store every default device (`complete: false`: a partial failure or a + * one-device render, whose other device pages are older than this record would claim) — stores + * null, which compares as no claim. A rule without fields keeps the original behaviour exactly: a + * render that yields no claim writes nothing. */ -export const recordPageClaim = async (url, structuredOffers, cachedAt = Date.now()) => { +export const recordPageClaim = async ( + url, + structuredOffers, + cachedAt = Date.now(), + { pageFacts = undefined, complete = true } = {} +) => { try { // The master switch gates STORAGE too — "Off = no probes, no timers, nothing stored" is // the config contract, and this is the hottest write path to be skipping work on. @@ -414,37 +451,64 @@ export const recordPageClaim = async (url, structuredOffers, cachedAt = Date.now // pageCheck-less rule shadow this URL on the sweep side: claims written per render here, // never compared there, and nothing to say so. const rule = probeRules().find((r) => r.pathPattern.test(pathname)); - if (!rule?.pageCheck) return; - if (structuredOffers === undefined) { - // The renderer does not know the field at all — it predates 1.20.0. There is - // deliberately NO fallback to parsing the stored HTML: recovering the offers here means - // a regex scan and a JSON parse of a ~1MB document on the hottest write path in this - // process, to reconstruct what the browser had structured in front of it. So pageCheck - // is INERT against an older renderer — say so rather than failing silently, since a - // config that looks enabled and protects nothing is the worst outcome. `null` is the - // other case and is NOT this warn: a >=1.20.0 renderer ran the extraction and the page - // declared no Product offers — nothing to record, same rule as a failed probe. - warnPageClaimUnsupported(); - return; + const pageCheck = rule?.pageCheck; + if (!pageCheck) return; + const claimPair = pageCheck.priceFrom !== null && pageCheck.priceFrom !== undefined; + const mapped = (pageCheck.fields?.length ?? 0) > 0; + if (!claimPair && !mapped) return; + let claim = null; + if (claimPair) { + if (structuredOffers === undefined) { + // The renderer does not know the field at all — it predates 1.20.0. There is + // deliberately NO fallback to parsing the stored HTML: recovering the offers here means + // a regex scan and a JSON parse of a ~1MB document on the hottest write path in this + // process, to reconstruct what the browser had structured in front of it. So pageCheck + // is INERT against an older renderer — say so rather than failing silently, since a + // config that looks enabled and protects nothing is the worst outcome. `null` is the + // other case and is NOT this warn: a >=1.20.0 renderer ran the extraction and the page + // declared no Product offers — nothing to record, same rule as a failed probe. + warnPageClaimUnsupported(); + } else { + claim = pageClaimFromOffers(structuredOffers); + } } - const claim = pageClaimFromOffers(structuredOffers); - if (!claim) return; - // patch cannot create, and put would clobber the probe's own signature/probedAt — so read - // first and choose. The read is a node-local point read on a small table, once per render - // of a pageCheck-matched URL. - const existing = await probeStateTable().get({ id: url, select: ['url'] }); // `pageClaimAt` rides in the SAME write — no extra read, no extra write — and is the render's // own `lastCached`, not the moment this ran. See the schema comment: it becomes the basis a // verification certifies, and a stamp taken milliseconds later would exclude the very page it // is certifying. const claimAt = new Date(cachedAt); - if (existing) await probeStateTable().patch(url, { pageSignature: claim, pageClaimAt: claimAt }); - else await probeStateTable().put(url, { url, pageSignature: claim, pageClaimAt: claimAt }); + let fields; + if (mapped) { + fields = { pageSignature: claim, pageFacts: pageRecordOf(url, pageFacts, complete), pageClaimAt: claimAt }; + } else { + if (!claim) return; + fields = { pageSignature: claim, pageClaimAt: claimAt }; + } + // patch cannot create, and put would clobber the probe's own signature/probedAt — so read + // first and choose. The read is a node-local point read on a small table, once per render + // of a pageCheck-matched URL. + const existing = await probeStateTable().get({ id: url, select: ['url'] }); + if (existing) await probeStateTable().patch(url, fields); + // A record of nothing is not worth creating a row for; on an EXISTING row the nulls above are + // the point (they retire an older render's record). + else if (fields.pageSignature !== null || fields.pageFacts) await probeStateTable().put(url, { url, ...fields }); } catch (e) { logger.warn?.(`[prerender] change-probe page claim not recorded for ${url}: ${e?.message ?? String(e)}`); } }; +/** The page record to store for one render, or null — see recordPageClaim for when and why. */ +const pageRecordOf = (url, pageFacts, complete) => { + if (pageFacts === undefined) { + warnPageFactsUnsupported(); + return null; + } + if (!complete) return null; + const { json, bytes, refused } = serializePageFacts(pageFacts); + if (refused) warnPageFactsRefused(url, bytes); + return json; +}; + // One line per hour per worker: this fires per RENDER, and a fleet mid-upgrade would otherwise // log it thousands of times a minute. let lastUnsupportedWarnAt = 0; @@ -459,6 +523,32 @@ const warnPageClaimUnsupported = () => { ); }; +// Same shape and cadence as the claim warning above, for the page record's own field. +let lastFactsUnsupportedWarnAt = 0; +const warnPageFactsUnsupported = () => { + const now = Date.now(); + if (now - lastFactsUnsupportedWarnAt < 3600000) return; + lastFactsUnsupportedWarnAt = now; + logger.warn?.( + `[prerender] changeProbe.pageCheck.fields are mapped but the render result carried no pageFacts — the ` + + `renderer is older than @harperfast/prerender-browser 1.37.0, so page records are stored empty and the ` + + `mapped fields detect nothing (and never suppress a trigger). Upgrade the render fleet or remove the mapping.` + ); +}; + +// Hourly too: an oversized record is a property of a page TEMPLATE, so one bad template would +// otherwise log on every render of every page built from it. +let lastFactsRefusedWarnAt = 0; +const warnPageFactsRefused = (url, bytes) => { + const now = Date.now(); + if (now - lastFactsRefusedWarnAt < 3600000) return; + lastFactsRefusedWarnAt = now; + logger.warn?.( + `[prerender] change-probe page record for ${url} is ${bytes} bytes, over the ${PAGE_FACTS_MAX_BYTES}-byte ` + + `bound — not stored, so that page's mapped fields compare as no claim. (Logged at most hourly.)` + ); +}; + /** * Probe a stream of target rows and act on what changed. ALL I/O is injected, so the decision * logic — ownership, matching, the seed/changed/deferred/failed state machine, pacing, the @@ -485,6 +575,132 @@ export const verificationArmedFor = async (scope) => { return (await resolveInvalidation(scope)) !== null; }; +/** + * THE MAPPING-DEFECT GUARD: disarm a mapped field that is evidently mapped wrong. + * + * A wrong mapping (a slot holding the regular price mapped to the sale price the page prints, a + * list compared in the wrong shape) disagrees on nearly EVERY comparison, and every disagreement is + * a re-render — so one bad mapping would re-render its whole corpus on every pass. A correct + * mapping disagrees only when the page really is wrong, and on a page the probe has WITNESSED — + * rendered after the stored baseline was taken, with the origin unchanged since — that takes a + * genuine round trip (the value changed and changed back between two probes, and a render landed in + * between): measured well under 1%. + * + * So the guard counts, per field, witnessed comparisons and witnessed disagreements, and DISARMS the + * field once its disagreement rate reaches `threshold` over at least `minWitnessed` comparisons. A + * disarmed field stops triggering (and stops counting toward caught-up and verification) but is + * still compared and still counted in `fieldMismatch`, and it stays disarmed until its mapping + * changes (a different entry, slot path or rule label is a different key) or the process restarts. + * + * AGGREGATE ONLY, BY DESIGN. It never suppresses an individual witnessed disagreement: a single one + * is exactly the round trip the page check exists to catch, and it must re-render. Only the RATE + * says "mapping defect", and only at a sample large enough that a correct mapping cannot reach it by + * chance (at a 1% true rate, 20% of 200 is ~40 disagreements where ~2 are expected). + * + * BOUNDED MEMORY. Both counts are halved whenever the sample reaches GUARD_WINDOW x `minWitnessed`, + * so the rate describes roughly the last few thousand comparisons rather than the process lifetime. + * Without it, a mapping that was right for a week and then broke — the site redesigns its title + * template, and every re-render still disagrees — would need hundreds of thousands of disagreements + * to outweigh the week of agreement, re-rendering every page it matches on every pass until then. + * The halved sample never drops below `minWitnessed`, so the threshold stays consultable. + * + * Per process, in memory: the sweep and canary run on worker 0, so that is where the evidence + * accumulates. A pass started by hand on another worker starts with its own, empty guard. + */ +// The guard's memory, in multiples of `minWitnessed` (see BOUNDED MEMORY above). +const GUARD_WINDOW = 10; + +export const createMappingGuard = ({ settings, onDisarm = () => {} }) => { + const entries = new Map(); + // A field's key, cached per compiled field object (a config reload compiles new ones). The rule + // label and the slot's extract PATH are in it, so re-pointing a slot or renaming the rule is a + // new mapping with a clean record. + const keys = new WeakMap(); + const keyOf = (rule, field) => { + let key = keys.get(field); + if (key === undefined) { + key = JSON.stringify([ + rule.label, + rule.extract?.[field.slot] ?? null, + field.slot, + field.fact, + field.compare, + field.options, + ]); + keys.set(field, key); + } + return key; + }; + return { + isArmed: (rule, field) => entries.get(keyOf(rule, field))?.disarmed !== true, + witness(rule, field, disagreed) { + const key = keyOf(rule, field); + let entry = entries.get(key); + if (!entry) entries.set(key, (entry = { witnessed: 0, disagreed: 0, disarmed: false })); + entry.witnessed++; + if (disagreed) entry.disagreed++; + if (entry.disarmed) return; + const { threshold, minWitnessed } = settings(); + if (entry.witnessed >= minWitnessed && entry.disagreed / entry.witnessed >= threshold) { + entry.disarmed = true; + onDisarm(rule, field, entry); + return; + } + if (entry.witnessed >= GUARD_WINDOW * minWitnessed) { + entry.witnessed /= 2; + entry.disagreed /= 2; + } + }, + /** `{ [rule]: { [field]: { witnessed, disagreed, armed } } }` for every mapped field of `rules`. */ + snapshot(rules) { + const out = {}; + for (const rule of rules) { + for (const field of rule.pageCheck?.fields ?? []) { + const entry = entries.get(keyOf(rule, field)); + out[rule.label] ??= {}; + out[rule.label][field.label] = { + witnessed: Math.round(entry?.witnessed ?? 0), + disagreed: Math.round(entry?.disagreed ?? 0), + armed: entry?.disarmed !== true, + }; + } + } + return out; + }, + }; +}; + +const warnDisarmed = (rule, field, entry) => { + logger.warn?.( + `[prerender] change-probe ${rule.label}: pageCheck field ${field.label} (extract[${field.slot}] ` + + `"${rule.extract?.[field.slot]}", compare ${field.compare}) is DISARMED — the cached page disagreed with ` + + `the origin on ${Math.round(entry.disagreed)} of ${Math.round(entry.witnessed)} recent witnessed comparisons ` + + `(${((entry.disagreed / entry.witnessed) * 100).toFixed(1)}%), where a correct mapping disagrees only on ` + + `rare round trips. It no longer triggers re-renders (its mismatches are still counted in fieldMismatch) ` + + `until the mapping is edited or the process restarts. Check that extract[${field.slot}] really holds what ` + + `the page shows as ${field.fact}.` + ); +}; + +// The node's guard — one per process, shared by the sweep and the canary (see createMappingGuard). +let mappingGuard = null; +const theMappingGuard = () => + (mappingGuard ??= createMappingGuard({ settings: () => config.changeProbe.mappingGuard, onDisarm: warnDisarmed })); + +// A Date column as epoch ms, BigInt-safe (see readSignature); NaN when unreadable. +const epochOf = (value) => { + if (value === undefined || value === null) return NaN; + const stamp = typeof value === 'bigint' ? Number(value) : value; + const ms = new Date(stamp).getTime(); + return Number.isFinite(ms) ? ms : NaN; +}; + +// `stats[group][rule][key]++`, creating the levels on first use. +const bump = (table, rule, key) => { + table[rule] ??= {}; + table[rule][key] = (table[rule][key] ?? 0) + 1; +}; + export const runProbePass = async ({ rows, rules, @@ -503,6 +719,9 @@ export const runProbePass = async ({ // resolved once per rule per pass (see below), never per row. verify = null, isArmed = null, + // The mapping-defect guard (createMappingGuard). Null = no guard: every mapped field is armed and + // nothing is witnessed, which is what a caller that does not wire it gets. + guard = null, dryRun, maxTriggers, concurrency, @@ -657,55 +876,142 @@ export const runProbePass = async ({ // exists to catch. Only for extracted responses — a status-signal literal carries no // price/availability to project (documented limitation). const verificationArmed = await isVerificationArmed(rule); + const pageCheck = rule.pageCheck; + const claimPair = pageCheck ? pageCheck.priceFrom !== null && pageCheck.priceFrom !== undefined : false; + const mappedFields = pageCheck?.fields ?? []; + + // The observation's slot values, parsed at most once and only when something needs them. + let values; + const valuesOf = () => { + if (values === undefined) values = signatureSlots(observed); // null for a status-signal literal + return values; + }; let pageDisagrees = false; - if (rule.pageCheck && stored?.pageSignature) { - let values = null; - try { - const parsed = JSON.parse(observed); - if (Array.isArray(parsed)) values = parsed; - } catch { - values = null; // a status-signal literal, not an extracted array - } - if (values) pageDisagrees = claimsDisagree(stored.pageSignature, apiClaimOf(values, rule.pageCheck)); - if (pageDisagrees) stats.pageMismatch++; + if (claimPair && stored?.pageSignature && valuesOf()) { + pageDisagrees = claimsDisagree(stored.pageSignature, apiClaimOf(values, pageCheck)); } const signatureChanged = Boolean(stored?.signature) && stored.signature !== comparable; + + // THE PAGE RECORD: every mapped field compared against what the cached page claims — the same + // question as the claim pair above, asked of every field the page visibly states. A + // disagreement triggers exactly as `pageDisagrees` does; an agreement is what lets a CHANGE be + // recognised as one the page has already caught up with (below). + let verdicts = null; + let facts = null; + let mappedDisagrees = false; + let mappedAgreed = 0; + const armed = (field) => !guard || guard.isArmed(rule, field); + if (mappedFields.length && stored?.pageFacts && valuesOf()) { + facts = parsePageFacts(stored.pageFacts); + if (facts) { + const ctx = { pageUrl: row.url, vocabulary: pageCheck.vocabulary ?? null }; + verdicts = mappedFields.map((field) => ({ + field, + verdict: compareField(field, values[field.slot], facts, ctx), + })); + // WITNESSED: the page was rendered after the baseline was taken, and the origin still says + // exactly what it said then — so a correct mapping MUST agree unless the value made a + // round trip in between. Only these comparisons feed the mapping-defect guard. An + // appended-path row is excluded: its new slots have no baseline to be unchanged against. + const witnessed = + !extended && Boolean(stored.signature) && !signatureChanged && epochOf(stored.pageClaimAt) > stored.probedAt; + for (const { field, verdict } of verdicts) { + if (verdict === null) continue; + if (witnessed) guard?.witness(rule, field, verdict === false); + if (verdict === false) { + bump(stats.fieldMismatch, rule.label, field.label); + if (armed(field)) mappedDisagrees = true; + } else if (armed(field)) { + mappedAgreed++; + } + } + } + } + if (pageDisagrees || mappedDisagrees) stats.pageMismatch++; + + // WHICH SLOTS CHANGED, and whether the change needs a render at all. Two ways it may not: + // + // IGNORED every changed slot is in `pageCheck.ignoreChanges` — fields the page cannot show. + // Baseline written, nothing triggered, and NOT a change for the canary: a site-wide + // edit of an ignored field (a badge, an inventory counter) must not read as a mass + // change and invalidate a route. + // CAUGHT UP every changed (non-ignored) slot is a mapped slot whose record ALREADY shows the + // new value — a cadence render landed after the change. Baseline written, nothing + // triggered; still a change for the canary (the rest of the corpus has not caught up). + // + // Anything else — an unmapped slot, a slot whose page record is unknown or disagrees, a + // status-signal literal (a state, not slots) — is a change, exactly as before. + // + // A slot is caught up only on positive evidence: at least one ARMED mapped field on it compared, + // and every armed field on it that compared shows the new value. A disarmed field is suspected of + // being mapped wrong, so its agreement is no evidence either way. + let storedValues = null; + const slotCaughtUp = (slot) => { + const onSlot = verdicts.filter(({ field, verdict }) => field.slot === slot && verdict !== null && armed(field)); + if (!onSlot.length) return false; + storedValues ??= signatureSlots(stored.signature) ?? []; + const ctx = { pageUrl: row.url, vocabulary: pageCheck.vocabulary ?? null }; + return onSlot.every(({ field }) => fieldCaughtUp(field, storedValues[slot], values[slot], facts, ctx)); + }; + let ignoredOnly = false; + let caughtUp = false; + if (signatureChanged) { + const slots = changedSlots(stored.signature, comparable); + if (slots === null) { + bump(stats.slotChanges, rule.label, 'signal'); + } else { + for (const slot of slots) bump(stats.slotChanges, rule.label, String(slot)); + const relevant = slots.filter((slot) => !(pageCheck?.ignoreChanges ?? []).includes(slot)); + if (!relevant.length) ignoredOnly = true; + else if (verdicts) caughtUp = relevant.every((slot) => slotCaughtUp(slot)); + } + } + + const needsRender = signatureChanged && !ignoredOnly && !caughtUp; // Bucket by SIGNATURE outcome alone, BEFORE the page check influences control flow: // `probed = seeded + rebaselined + unchanged + changed + failed` is the documented invariant, and the // canary's denominator is changed + unchanged — a page-mismatch row that skipped both // would silently shrink the mass-change sample right when claims are most likely to be - // stale. `pageMismatch` OVERLAYS these buckets; it never replaces them. - if (signatureChanged) stats.changed++; + // stale. `pageMismatch`, `caughtUp` and `ignored` OVERLAY these buckets; they never replace + // them. An IGNORED change is bucketed unchanged — for the canary it is not a change. + if (signatureChanged && !ignoredOnly) stats.changed++; else if (stored?.signature) stats.unchanged++; else stats.seeded++; - if (!signatureChanged && !pageDisagrees) { + if (ignoredOnly) stats.ignored++; + if (caughtUp) stats.caughtUp++; + if (!needsRender && !pageDisagrees && !mappedDisagrees) { if (!stored?.signature) { // First observation: baseline it, trigger nothing — the page's content is not known // to have changed, the probe just hadn't seen it before. await write(row.url, observed, { rowExists: stored !== null, fingerprint: rule.fingerprint }); return; } - // One-time stamp of a pre-fingerprint row (see the rule-changed block above), or one-time - // upgrade of a baseline taken before paths were appended — the full observation replaces - // the shorter signature, so the next pass compares every slot. One patch per such row, - // after which a converged corpus is back to paying no write per probe. + // The new baseline of an ignored or caught-up change; or the one-time stamp of a + // pre-fingerprint row (see the rule-changed block above), or one-time upgrade of a baseline + // taken before paths were appended — the full observation replaces the shorter signature, so + // the next pass compares every slot. One patch per such row, after which a converged corpus + // is back to paying no write per probe. // Not an alternative to the verification below — both are due, neither stands in for the other. - if (extended || stored.fingerprint === null || stored.fingerprint === undefined) { + if (signatureChanged || extended || stored.fingerprint === null || stored.fingerprint === undefined) { await write(row.url, observed, { rowExists: true, fingerprint: rule.fingerprint }); } - if (verificationArmed && stored.pageSignature) { + // PROOF that something was compared: the stored claim for a rule with the claim pair (the + // original gate, unchanged), else at least one armed mapped field that agreed. A caught-up + // row is not verified on this pass — its baseline just moved — and is on the next. + const proof = claimPair ? Boolean(stored.pageSignature) : mappedAgreed > 0; + if (verificationArmed && proof && !caughtUp && !mappedDisagrees) { // PROOF, not absence of news. Both conditions are load-bearing and neither is // redundant: // - // stored.pageSignature `pageDisagrees` is computed only `if (rule.pageCheck && - // stored?.pageSignature)`, so a URL whose page claim is unknown - // arrives here with `pageDisagrees === false` — identical, at - // this line, to a real agreement. Without this guard we would - // stamp "verified" on a page nobody ever compared, and serve it - // through an invalidation. That is the one failure this feature - // must never produce. + // proof `pageDisagrees` is computed only when a stored claim exists (and + // a mapped field only when a stored record does), so a URL whose + // page claim is unknown arrives here with nothing disagreeing — + // identical, at this line, to a real agreement. Without this guard + // we would stamp "verified" on a page nobody ever compared, and + // serve it through an invalidation. That is the one failure this + // feature must never produce. // // verificationArmed the exemption only means anything while an invalidation is // active for THIS RULE'S scope, and a converged corpus is @@ -713,6 +1019,9 @@ export const runProbePass = async ({ // comment). Writing unconditionally would add ~200k writes per // node per cycle in the steady state to buy nothing. // + // `!mappedDisagrees` is structurally true here (a disagreement leaves this branch), and is + // stated anyway: no armed mapped field may disagree with a page this certifies. + // // Awaited, not detached: it shares the pass's pacing budget, and a write storm that // outruns the probe is exactly what the paced sweep exists to prevent. await verify(row.url, stored.pageClaimAt); @@ -852,6 +1161,10 @@ export const runProbePass = async ({ } await flush(); + // Where each mapped field's guard stands after this pass (cumulative for the process, not the + // pass): how close a mapping is to being disarmed, and which ones are. Only for rules that map. + if (guard && rules.some((rule) => rule.pageCheck?.fields?.length)) stats.fieldGuard = guard.snapshot(rules); + return stats; }; @@ -1120,6 +1433,7 @@ export const runProbeSweepOnce = async ({ dryRun, label = null, reseed = false } submitTrigger: triggers.submit, verify: writeVerification, isArmed: verificationArmedFor, + guard: theMappingGuard(), ...limits, // The liveness signal for the node-wide claim, throttled inside `makeHeartbeat` — this // fires every YIELD_EVERY rows, the heartbeat writes at most every HEARTBEAT_MS. Its @@ -1357,6 +1671,12 @@ export const runProbeCanaryOnce = async ({ dryRun } = {}) => { // IMMEDIATE, not queued: the cohort is a few hundred URLs and the canary's whole value // is being fast, so there is nothing for a paced queue to spread. submitTrigger: canaryTriggers.submit, + // The SAME guard as the sweep, reading AND witnessing: the canary re-probes its cohort + // every few minutes, so a broken mapping would otherwise re-render the whole cohort on + // every canary pass until the (possibly daily) sweep got round to disarming it. Repeat + // witnesses of one cohort do not skew the rate — they repeat its agreements and its + // disagreements alike. + guard: theMappingGuard(), ...limits, // NEVER skips on baseline age. The cohort is small and deliberately probed on a // cadence far tighter than `reprobeAfter` — freshness-skipping here would silence @@ -1442,6 +1762,12 @@ export const changeProbeStatus = async () => { pathPattern: rule.patternSource, source: rule.source, invalidateScope: rule.invalidateScope, + // Only when present, so a rule without them reads exactly as it did — the console compares + // these objects across nodes to spot a rule list that did not reach every node. + ...(rule.pageCheck?.fields?.length + ? { pageFields: rule.pageCheck.fields.map((field) => `${field.label}:${field.compare}`) } + : {}), + ...(rule.pageCheck?.ignoreChanges?.length ? { ignoreChanges: rule.pageCheck.ignoreChanges } : {}), })), mode: config.changeProbe.mode, // False only when the row could not be read at all. Distinguishes "the probe has not run @@ -1748,6 +2074,9 @@ export const publishProbeStateForTest = publishProbeState; /** Tests only — the limits builder, so the sweep/canary split is assertable without a live pass. */ export const __passLimitsForTest = passLimits; +/** Tests only — the process's mapping guard, as the sweep and canary get it (live config, real warning). */ +export const __mappingGuardForTest = theMappingGuard; + /** Tests only — module state that outlives a beforeEach. */ export const resetChangeProbeState = () => { clearProbeTimers(); @@ -1761,6 +2090,9 @@ export const resetChangeProbeState = () => { compiledRules = null; compiledFrom = undefined; lastUnsupportedWarnAt = 0; + lastFactsUnsupportedWarnAt = 0; + lastFactsRefusedWarnAt = 0; + mappingGuard = null; measuredSliceSize = null; stopLoopLagMonitor(); }; diff --git a/packages/plugin/src/util/changeProbeSpec.js b/packages/plugin/src/util/changeProbeSpec.js index 31d34116..b53204c9 100644 --- a/packages/plugin/src/util/changeProbeSpec.js +++ b/packages/plugin/src/util/changeProbeSpec.js @@ -98,6 +98,16 @@ const compileRule = (raw, index, warn) => { warn(`change-probe ${label}: source "request" requires extract — a non-empty array of value paths`); return null; } + // A malformed tuple projection would extract null on every probe, forever — a slot that looks + // configured and watches nothing. Refused here, where the operator sees it, like any other + // rule defect. + for (const [i, path] of raw.extract.entries()) { + const problem = extractPathProblem(path); + if (problem) { + warn(`change-probe ${label}: extract[${i}] "${path}": ${problem}`); + return null; + } + } request = { urlTemplate: raw.request.urlTemplate, method, headers, body }; extract = raw.extract.slice(); } else { @@ -165,54 +175,21 @@ const compileRule = (raw, index, warn) => { // Which extracted values correspond to what the PAGE renders, so the probe can ask "does the // cached page still agree with the origin" as well as "did the origin change". Site-specific // by nature: only the operator knows which field of their endpoint is the price the page - // prints. Indices into `extract`, so nothing new is fetched. Dropped whole (not per-field) — - // a half-configured mapping would compare the wrong column. + // prints. Indices into `extract`, so nothing new is fetched. See `compilePageCheck`. let pageCheck = null; if (raw.pageCheck !== undefined && raw.pageCheck !== null) { const pc = raw.pageCheck; if (typeof pc !== 'object' || Array.isArray(pc)) { - warn(`change-probe ${label}: pageCheck must be an object { enabled, priceFrom, availableFrom }`); + warn( + `change-probe ${label}: pageCheck must be an object { enabled, priceFrom, availableFrom, fields, ignoreChanges }` + ); } else if (pc.enabled === true) { if (source !== 'request') { // In document mode the stored signature IS the page's own offers, so the page can // never disagree with itself and the comparison is meaningless. warn(`change-probe ${label}: pageCheck applies to source "request" only — ignored`); } else { - const inBounds = (v) => Number.isInteger(v) && v >= 0 && v < extract.length; - if (!inBounds(pc.priceFrom) || !inBounds(pc.availableFrom)) { - warn( - `change-probe ${label}: pageCheck.priceFrom and .availableFrom must be integer indices into ` + - `extract (0-${extract.length - 1}) — pageCheck ignored` - ); - } else { - pageCheck = { priceFrom: pc.priceFrom, availableFrom: pc.availableFrom, vocabulary: null }; - // The rule's own availability words, on top of the built-in schema.org/retail sets. - // Tokenized the same way the endpoint's values will be, so "In Stock" and "IN_STOCK" in - // config meet "in stock" in a response. Invalid lists drop the vocabulary, not pageCheck: - // the built-in words still apply. - const words = (key) => { - const list = pc[key]; - if (list === undefined || list === null) return []; - if (!Array.isArray(list) || list.some((w) => typeof w !== 'string' || !availabilityToken(w))) { - warn(`change-probe ${label}: pageCheck.${key} must be an array of availability words — ignored`); - return null; - } - return list.map(availabilityToken); - }; - const available = words('availableValues'); - const unavailable = words('unavailableValues'); - if (available && unavailable && (available.length || unavailable.length)) { - const overlap = available.filter((w) => unavailable.includes(w)); - if (overlap.length) { - warn( - `change-probe ${label}: pageCheck.availableValues and .unavailableValues both list ` + - `${overlap.join(', ')} — vocabulary ignored` - ); - } else { - pageCheck.vocabulary = { available: new Set(available), unavailable: new Set(unavailable) }; - } - } - } + pageCheck = compilePageCheck(pc, extract, label, warn); } } else if (pc.enabled) { // `enabled: "true"` (a YAML/JSON string) must not silently disable: a config that LOOKS @@ -242,6 +219,186 @@ const compileRule = (raw, index, warn) => { return rule; }; +/** + * Compile an ENABLED request-mode `pageCheck` block, or null when nothing in it is usable. + * + * Three independent parts, each validated on its own so one bad part never costs the others: + * + * priceFrom/availableFrom the original price + availability claim, compiled EXACTLY as before — + * dropped as a pair if either index is out of bounds, because a + * half-applied pair compares the wrong column. Optional once `fields` + * or `ignoreChanges` is given (null here then means "no claim pair"). + * fields the page-record mapping (`compilePageFields`): a bad entry drops that + * entry alone, with a warning. + * ignoreChanges slots whose origin changes never trigger (`compileIgnoreChanges`). + * + * The vocabulary (availableValues/unavailableValues) serves both the pair and a `skus` field, so it + * compiles whenever the block survives. + */ +const compilePageCheck = (pc, extract, label, warn) => { + const inBounds = (v) => Number.isInteger(v) && v >= 0 && v < extract.length; + const fields = compilePageFields(pc.fields, inBounds, extract, label, warn); + const ignoreChanges = compileIgnoreChanges(pc.ignoreChanges, inBounds, extract, label, warn); + const more = fields.length > 0 || ignoreChanges.length > 0; + let priceFrom = null; + let availableFrom = null; + // Without fields or ignoreChanges the pair is the whole block, so it is required exactly as it + // always was. With them it is optional — but a pair that IS given must still be a valid one. + if ( + !more || + (pc.priceFrom !== undefined && pc.priceFrom !== null) || + (pc.availableFrom !== undefined && pc.availableFrom !== null) + ) { + if (!inBounds(pc.priceFrom) || !inBounds(pc.availableFrom)) { + warn( + `change-probe ${label}: pageCheck.priceFrom and .availableFrom must be integer indices into ` + + `extract (0-${extract.length - 1}) — ` + + (more + ? 'the price/availability claim is ignored (fields and ignoreChanges still apply)' + : 'pageCheck ignored') + ); + if (!more) return null; + } else { + priceFrom = pc.priceFrom; + availableFrom = pc.availableFrom; + } + } + return { priceFrom, availableFrom, vocabulary: compileVocabulary(pc, label, warn), fields, ignoreChanges }; +}; + +/** + * The rule's own availability words, on top of the built-in schema.org/retail sets. Tokenized the + * same way the endpoint's values will be, so "In Stock" and "IN_STOCK" in config meet "in stock" in + * a response. Invalid lists drop the vocabulary, not pageCheck: the built-in words still apply. + */ +const compileVocabulary = (pc, label, warn) => { + const words = (key) => { + const list = pc[key]; + if (list === undefined || list === null) return []; + if (!Array.isArray(list) || list.some((w) => typeof w !== 'string' || !availabilityToken(w))) { + warn(`change-probe ${label}: pageCheck.${key} must be an array of availability words — ignored`); + return null; + } + return list.map(availabilityToken); + }; + const available = words('availableValues'); + const unavailable = words('unavailableValues'); + if (!available || !unavailable || (!available.length && !unavailable.length)) return null; + const overlap = available.filter((w) => unavailable.includes(w)); + if (overlap.length) { + warn( + `change-probe ${label}: pageCheck.availableValues and .unavailableValues both list ` + + `${overlap.join(', ')} — vocabulary ignored` + ); + return null; + } + return { available: new Set(available), unavailable: new Set(unavailable) }; +}; + +/** + * `pageCheck.fields`: which extracted slot holds what the page shows as which PAGE FACT, and how + * the two are compared. Each entry `{ slot, fact, compare, ...options }` compiles to + * `{ slot, fact, compare, options, label }` where `label` (":") names it in stats and + * warnings. + * + * DROPPED PER ENTRY, never the rule and never the list: a mapping entry is independent of its + * siblings (each compares its own slot to its own fact), so a typo in one is no reason to stop + * checking the others. What IS refused is an entry that could only ever compare wrongly — a + * comparator applied to a fact of the wrong kind (a price set against a title), an unknown fact or + * comparator, an out-of-bounds slot — because a confident wrong comparison disagrees on every pass + * and re-renders every page it matches, forever. + */ +const compilePageFields = (list, inBounds, extract, label, warn) => { + if (list === undefined || list === null) return []; + if (!Array.isArray(list)) { + warn(`change-probe ${label}: pageCheck.fields must be an array of { slot, fact, compare } — ignored`); + return []; + } + const fields = []; + const seen = new Set(); + for (const [i, entry] of list.entries()) { + const where = `change-probe ${label}: pageCheck.fields[${i}]`; + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + warn(`${where} must be an object { slot, fact, compare } — entry dropped`); + continue; + } + if (!inBounds(entry.slot)) { + warn(`${where}.slot must be an integer index into extract (0-${extract.length - 1}) — entry dropped`); + continue; + } + if (typeof entry.fact !== 'string' || !Object.hasOwn(PAGE_FACTS, entry.fact)) { + warn(`${where}.fact must be one of ${Object.keys(PAGE_FACTS).join(', ')} — entry dropped`); + continue; + } + if (typeof entry.compare !== 'string' || !Object.hasOwn(COMPARATORS, entry.compare)) { + warn(`${where}.compare must be one of ${Object.keys(COMPARATORS).join(', ')} — entry dropped`); + continue; + } + const comparator = COMPARATORS[entry.compare]; + const kind = PAGE_FACTS[entry.fact].kind; + if (!comparator.kinds.includes(kind)) { + const fitting = Object.keys(COMPARATORS).filter((name) => COMPARATORS[name].kinds.includes(kind)); + warn( + `${where}: compare "${entry.compare}" does not apply to fact "${entry.fact}" (use ${fitting.join(' or ')}) — entry dropped` + ); + continue; + } + let problem = null; + const options = comparator.options(entry, (message) => { + problem = message; + }); + if (problem !== null) { + warn(`${where}: ${problem} — entry dropped`); + continue; + } + const unknown = Object.keys(entry).filter( + (key) => key !== 'slot' && key !== 'fact' && key !== 'compare' && !comparator.keys.includes(key) + ); + if (unknown.length) warn(`${where}: unknown key(s) ${unknown.join(', ')} ignored`); + const fieldLabel = `${entry.slot}:${entry.fact}`; + if (seen.has(fieldLabel)) { + warn(`${where} maps slot ${entry.slot} to ${entry.fact} a second time — entry dropped`); + continue; + } + seen.add(fieldLabel); + fields.push({ slot: entry.slot, fact: entry.fact, compare: entry.compare, options, label: fieldLabel }); + } + return fields; +}; + +/** + * `pageCheck.ignoreChanges`: extract slots whose origin changes do NOT trigger a re-render — the + * fields the page cannot show (an inventory counter, a store flag), watched only because the + * endpoint returns them next to fields it can. A change confined to these slots writes the new + * baseline and triggers nothing, and the canary does not count it as a change either. Returned + * sorted and de-duplicated; a bad entry drops alone. + */ +const compileIgnoreChanges = (list, inBounds, extract, label, warn) => { + if (list === undefined || list === null) return []; + if (!Array.isArray(list)) { + warn(`change-probe ${label}: pageCheck.ignoreChanges must be an array of extract indices — ignored`); + return []; + } + const slots = new Set(); + for (const [i, slot] of list.entries()) { + if (!inBounds(slot)) { + warn( + `change-probe ${label}: pageCheck.ignoreChanges[${i}] must be an integer index into extract ` + + `(0-${extract.length - 1}) — entry dropped` + ); + continue; + } + slots.add(slot); + } + if (slots.size === extract.length) { + warn( + `change-probe ${label}: pageCheck.ignoreChanges lists EVERY extract slot — no origin change on this rule ` + + `will ever trigger a re-render; only page mismatches (pageCheck fields) will` + ); + } + return [...slots].sort((a, b) => a - b); +}; + /** * What a rule OBSERVES, hashed: the endpoint, how it is asked, which values are taken from the * answer, and which statuses stand in for values. Stored beside every baseline (ProbeState. @@ -494,7 +651,11 @@ const availabilityClaim = (raw, vocabulary) => { * Null when neither field yields a claim. */ export const apiClaimOf = (values, pageCheck) => { - if (!pageCheck || !Array.isArray(values)) return null; + // A block with no claim pair (fields or ignoreChanges only) projects no claim at all — an index + // of null would read `values[null]` and happen to yield nothing, which is not a contract. + if (!pageCheck || !Array.isArray(values) || pageCheck.priceFrom === null || pageCheck.priceFrom === undefined) { + return null; + } const price = canonicalPrice(values[pageCheck.priceFrom]); const available = availabilityClaim(values[pageCheck.availableFrom], pageCheck.vocabulary ?? null); if (price === null && available === null) return null; @@ -536,6 +697,489 @@ export const claimsDisagree = (pageClaim, apiClaim) => { } }; +// ---- the page record: what each cached page claims, field by field ------------------------------ + +/** + * THE PAGE RECORD generalizes the price/availability claim above to every field a page visibly + * states. The renderer reads them off the settled DOM (`pageFacts`, @harperfast/prerender-browser + * >= 1.37.0) and posts them with the result; the render path stores them beside the claim + * (`ProbeState.pageFacts`, see changeProbe.js recordPageClaim); and each probe compares them with + * the slots a rule maps (`pageCheck.fields`). That lets the probe ask "is the CACHED PAGE wrong" + * instead of only "did the ORIGIN change" — so it re-renders a page that disagrees on any mapped + * field, and does NOT re-render one that already shows the new value (a cadence render that landed + * after the change). + * + * EVERY COMPARATOR FAILS TOWARD NO CLAIM, NEVER A GUESS. Null, absent, empty or unparseable on + * either side is "not compared" (null), never a disagreement: a confident wrong comparison + * disagrees on every pass and re-renders every page it matches, forever — the one failure this + * feature must not have. The comparators are a CLOSED set of typed functions rather than + * configurable transforms for the same reason: each one encodes a measured quirk of how an + * endpoint and a page state the same fact, and nothing else. + */ + +/** Largest page record stored, in UTF-8 bytes — see `serializePageFacts`. */ +export const PAGE_FACTS_MAX_BYTES = 16 * 1024; + +/** + * The page facts a mapping may name, with the KIND of value each holds. The kind decides which + * comparators apply, so a nonsensical pairing (a price set against a title) is refused at compile + * time instead of disagreeing at run time. + */ +const PAGE_FACTS = { + 'canonical': { kind: 'url', get: (facts) => facts.canonical }, + 'title': { kind: 'text', get: (facts) => facts.title }, + 'metaDescription': { kind: 'text', get: (facts) => facts.metaDescription }, + 'h1': { kind: 'text', get: (facts) => facts.h1 }, + 'product.name': { kind: 'text', get: (facts) => facts.product?.name }, + 'product.brand': { kind: 'text', get: (facts) => facts.product?.brand }, + 'product.image': { kind: 'url', get: (facts) => facts.product?.image }, + 'product.rating.value': { kind: 'number', get: (facts) => facts.product?.rating?.[0] }, + 'product.rating.count': { kind: 'number', get: (facts) => facts.product?.rating?.[1] }, + 'product.offers': { kind: 'offers', get: (facts) => facts.product?.offers }, + 'breadcrumbs': { kind: 'names', get: (facts) => facts.breadcrumbs }, +}; + +/** + * Text as a page and an endpoint both state it: NFC, every whitespace run (NBSP and line breaks + * included) collapsed to one space, trimmed. Deliberately NOT case-folded and NOT HTML-stripped — + * measured, an endpoint's meta description can carry `
` and `
  • ` while the page's + * `` holds that exact raw string, so stripping would turn a 100% match into + * a disagreement. Non-strings and empty text are no claim. + */ +const normalizeText = (value) => { + if (typeof value !== 'string') return null; + const text = value.normalize('NFC').replace(/\s+/g, ' ').trim(); + return text === '' ? null : text; +}; + +/** + * A URL's PATH, the only part of it an endpoint and a page reliably agree on: measured, the + * endpoint's image URL differed from the page's on every page, only by size query parameters. + * Origin, query and fragment are ignored; a relative value resolves against the page's own URL. + * `decodeURI` (not `decodeURIComponent`) so `%7E` meets `~` without `%2F` turning into a separator. + */ +const pathOf = (value, base) => { + if (typeof value !== 'string' || value.trim() === '') return null; + const url = URL.parse(value.trim(), base ?? undefined); + if (!url) return null; + try { + return decodeURI(url.pathname); + } catch { + return url.pathname; + } +}; + +/** A finite number from a number or a numeric string ("4.0" from an endpoint, 4 from JSON-LD). */ +const numberOf = (value) => { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value !== 'string' || value.trim() === '') return null; + const n = Number(value.trim()); + return Number.isFinite(n) ? n : null; +}; + +/** + * The distinct canonical prices of a value, or null when ANY element is unreadable: a set with a + * hole in it cannot be compared for equality, and guessing the hole away would disagree with the + * page on every pass for that product. Measured, an endpoint's single "lowest price" is null + * exactly when the product is out of stock, so a null here is "no claim" and nothing more. + */ +const priceSetOf = (list) => { + if (!Array.isArray(list) || !list.length) return null; + const prices = new Set(); + for (const item of list) { + const price = canonicalPrice(item); + if (price === null) return null; + prices.add(price); + } + return prices; +}; + +/** A SKU key: a non-empty string, or a number in its string form (the renderer's convention). */ +const skuKey = (value) => { + if (typeof value === 'string') return value.trim() || null; + if (typeof value === 'number' && Number.isFinite(value)) return String(value); + return null; +}; + +/** + * Per-SKU state keyed by SKU, with every SKU that appears more than once set aside in `dups`: two + * different states for one key is ambiguous, and ambiguity is no claim. + */ +const keyedBySku = (entries, read) => { + const map = new Map(); + const dups = new Set(); + for (const entry of Array.isArray(entries) ? entries : []) { + if (!Array.isArray(entry)) continue; + const state = read(entry); + if (state.sku === null) continue; + if (map.has(state.sku) || dups.has(state.sku)) { + map.delete(state.sku); + dups.add(state.sku); + continue; + } + map.set(state.sku, state); + } + return { map, dups }; +}; + +// The endpoint's tuples under a field's `tuple` naming, and the page's [sku, price, currency, +// availability] offers, both as { sku, price, availability }. +const apiSkus = (value, tuple) => + keyedBySku(value, (entry) => ({ + sku: skuKey(entry[tuple.sku]), + price: tuple.price === null ? undefined : entry[tuple.price], + availability: tuple.availability === null ? undefined : entry[tuple.availability], + })); +const pageSkus = (offers) => + keyedBySku(offers, (offer) => ({ sku: skuKey(offer[0]), price: offer[1], availability: offer[3] })); + +/** + * One SKU on both sides: false when the availability VERDICTS differ (both must be readable — the + * same tri-state reduction the claim pair uses, the rule's vocabulary on the endpoint side) or, + * when both state one, the prices differ; true when at least one of those was compared and agreed; + * null when neither could be. + */ +const skuVerdict = (api, page, tuple, vocabulary) => { + let compared = false; + if (tuple.availability !== null) { + const apiInStock = availabilityClaim(api.availability, vocabulary); + const pageInStock = availabilityVerdict(page.availability); + if (typeof apiInStock === 'boolean' && typeof pageInStock === 'boolean') { + if (apiInStock !== pageInStock) return false; + compared = true; + } + } + if (tuple.price !== null) { + const apiPrice = canonicalPrice(api.price); + const pagePrice = canonicalPrice(page.price); + if (apiPrice !== null && pagePrice !== null) { + if (apiPrice !== pagePrice) return false; + compared = true; + } + } + return compared ? true : null; +}; + +const noOptions = () => ({}); + +/** + * The closed set of comparators. `kinds` are the fact kinds each applies to, `keys` the extra + * entry keys it reads, `options` validates them (calling `fail` on a bad one), and `compare` + * returns true (agrees), false (disagrees) or null (not compared). + */ +const COMPARATORS = { + // Exact after normalizeText. Measured 100% on title, product name, h1, meta description, brand. + text: { + kinds: ['text', 'url'], + keys: [], + options: noOptions, + compare: (api, page) => { + const a = normalizeText(api); + const b = normalizeText(page); + return a === null || b === null ? null : a === b; + }, + }, + // URL pathname only. Measured 100% on an SEO URL against the canonical's path; required for + // images, whose URLs differ on every page by size parameters alone. + path: { + kinds: ['url'], + keys: [], + options: noOptions, + compare: (api, page, field, ctx) => { + const a = pathOf(api, ctx.pageUrl); + const b = pathOf(page, ctx.pageUrl); + return a === null || b === null ? null : a === b; + }, + }, + // Numeric equality with an optional absolute tolerance: an endpoint's rating is "4.0", the + // page's 4. + number: { + kinds: ['number'], + keys: ['tolerance'], + options: (entry, fail) => { + const tolerance = entry.tolerance === undefined || entry.tolerance === null ? 0 : entry.tolerance; + if (typeof tolerance !== 'number' || !Number.isFinite(tolerance) || tolerance < 0) { + fail('tolerance must be a finite number >= 0'); + } + return { tolerance }; + }, + compare: (api, page, field) => { + const a = numberOf(api); + const b = numberOf(page); + return a === null || b === null ? null : Math.abs(a - b) <= field.options.tolerance + 1e-9; + }, + }, + // The endpoint's price(s) against the SET of prices the page's offers print (distinct canonical + // 2-decimal strings), asked in the only direction a page can answer for itself: + // + // a single endpoint price the page must print it (the claim pair's rule: a page may list + // several variant prices while the endpoint reports one); + // a list of them EVERY price the page prints must still be in the endpoint's set. + // + // NOT set equality, and the difference is load-bearing. A page may list only SOME variants — + // measured, one origin's structured data stops at 50 offers while its endpoint lists every + // variant — so a price that exists only on an unlisted variant is absent from the page by + // construction. Equality would disagree on that product on every pass, re-render it, and + // disagree again with the re-render: a perpetual re-render loop the mapping guard cannot see, + // because the affected products are a small minority. What the subset test gives up is a new + // price on a listed variant while its old price survives on another; per-variant exactness is + // what `skus` is for. + priceSet: { + kinds: ['offers'], + keys: [], + options: noOptions, + compare: (api, page) => { + const a = priceSetOf(Array.isArray(api) ? api : [api]); + const b = Array.isArray(page) ? priceSetOf(page.map((offer) => (Array.isArray(offer) ? offer[1] : null))) : null; + if (a === null || b === null) return null; + if (!Array.isArray(api)) return b.has([...a][0]); + return [...b].every((price) => a.has(price)); + }, + }, + // An ordered list of names (strings, or objects carrying `nameKey`, default "name") against the + // page's list, as a SUFFIX: measured, a page's breadcrumbs open with a site-home crumb the + // endpoint omits, so the endpoint's list is compared with the page list's tail of the same + // length — no per-site configuration. An endpoint list LONGER than the page's is a disagreement + // (the page is missing levels). Extract the list itself, not a `[*]` projection: projections are + // sorted, and order is what this compares. + names: { + kinds: ['names'], + keys: ['nameKey'], + options: (entry, fail) => { + const nameKey = entry.nameKey === undefined || entry.nameKey === null ? 'name' : entry.nameKey; + if (typeof nameKey !== 'string' || nameKey === '') fail('nameKey must be a non-empty string'); + return { nameKey }; + }, + compare: (api, page, field) => { + if (!Array.isArray(api) || !api.length || !Array.isArray(page) || !page.length) return null; + const names = []; + for (const item of api) { + const raw = item && typeof item === 'object' && !Array.isArray(item) ? item[field.options.nameKey] : item; + const name = normalizeText(raw); + if (name === null) return null; + names.push(name); + } + const crumbs = page.map(normalizeText); + if (crumbs.includes(null)) return null; + if (names.length > crumbs.length) return false; + const tail = crumbs.slice(crumbs.length - names.length); + return names.every((name, i) => name === tail[i]); + }, + }, + // Per-variant state: the endpoint's tuples (a `[*].{…}` projection) against the page's offers, + // keyed by SKU, over the INTERSECTION of SKUs only — measured, a page lists at most 50 SKU offers + // where the endpoint lists more, so a SKU missing from either side is not a disagreement. + // `tuple` names each tuple position: "sku" (required), "availability", "price", or null to skip + // a position; default ["sku", "availability", "price"]. + skus: { + kinds: ['offers'], + keys: ['tuple'], + options: (entry, fail) => { + const names = entry.tuple === undefined || entry.tuple === null ? ['sku', 'availability', 'price'] : entry.tuple; + const tuple = { sku: null, availability: null, price: null }; + if (!Array.isArray(names) || !names.length) { + fail('tuple must be an array naming each tuple position ("sku", "availability", "price" or null)'); + return null; + } + for (const [i, name] of names.entries()) { + if (name === null) continue; + if (!Object.hasOwn(tuple, name) || tuple[name] !== null) { + fail(`tuple[${i}] must be "sku", "availability", "price" or null, each at most once`); + return null; + } + tuple[name] = i; + } + if (tuple.sku === null || (tuple.availability === null && tuple.price === null)) { + fail('tuple must name "sku" and at least one of "availability" or "price"'); + return null; + } + return { tuple }; + }, + compare: (api, page, field, ctx) => { + if (!Array.isArray(api) || !Array.isArray(page)) return null; + const { tuple } = field.options; + const endpoint = apiSkus(api, tuple).map; + const offers = pageSkus(page).map; + let agreed = false; + for (const [sku, state] of endpoint) { + const offer = offers.get(sku); + if (!offer) continue; + const verdict = skuVerdict(state, offer, tuple, ctx.vocabulary); + if (verdict === false) return false; + if (verdict === true) agreed = true; + } + return agreed ? true : null; + }, + }, +}; + +/** + * One mapped field's verdict: true (the page agrees with the endpoint value), false (disagrees), + * null (not compared). `facts` is a parsed page record (`parsePageFacts`), `ctx` is + * `{ pageUrl, vocabulary }`. Never throws — a comparator fault on data from a previous release or + * a corrupted row must not end the sweep; it is no claim. + */ +export const compareField = (field, apiValue, facts, ctx = {}) => { + if (!facts || apiValue === null || apiValue === undefined) return null; + try { + const pageValue = PAGE_FACTS[field.fact].get(facts); + if (pageValue === null || pageValue === undefined) return null; + return COMPARATORS[field.compare].compare(apiValue, pageValue, field, ctx); + } catch { + return null; + } +}; + +/** + * Has the page CAUGHT UP with a change in this field's slot — does it already show `after`, the + * value the probe just observed in place of `before`? True only on positive evidence, because a + * true here is what lets a real change go un-rendered. + * + * THE PAGE MUST AGREE WITH THE NEW VALUE AND NOT WITH THE OLD ONE. Agreement with `after` alone is + * not evidence when the change is invisible to the comparator: a crumb's URL changing inside a + * `names` object (only names are compared), a query parameter under `path`, a whitespace edit under + * `text`, a variant removed from a `priceSet` without changing the set. The page agrees with both + * values then, so nothing says it was re-rendered since — and the part that changed may well be on + * the page (a crumb's link) and stale. Those changes trigger, exactly as they did before fields. + * + * `skus` compares an intersection, so even that proves nothing about the SKU that actually changed — + * it may be one the page does not list (the page's offers are truncated), and reading "the SKUs I + * can see agree" as "the page caught up" would swallow that change. So every SKU whose tuple changed + * is proven on its own: a SKU still at the endpoint must be on the page, agreeing with its new state + * and not its old one (a change only in a skipped tuple position proves nothing); a SKU gone from the + * endpoint must be gone from the page. A changed tuple that cannot be keyed (no SKU, or a duplicated + * one) is not provable at all. + */ +export const fieldCaughtUp = (field, before, after, facts, ctx = {}) => { + if (compareField(field, after, facts, ctx) !== true) return false; + if (compareField(field, before, facts, ctx) === true) return false; + if (field.compare !== 'skus') return true; + try { + const { tuple } = field.options; + const was = apiSkus(before, tuple); + const now = apiSkus(after, tuple); + const page = pageSkus(PAGE_FACTS['product.offers'].get(facts)); + const beforeTuples = new Set((Array.isArray(before) ? before : []).map((entry) => JSON.stringify(entry))); + const afterTuples = new Set((Array.isArray(after) ? after : []).map((entry) => JSON.stringify(entry))); + const changed = [ + ...(Array.isArray(after) ? after : []).filter((entry) => !beforeTuples.has(JSON.stringify(entry))), + ...(Array.isArray(before) ? before : []).filter((entry) => !afterTuples.has(JSON.stringify(entry))), + ]; + for (const entry of changed) { + const sku = Array.isArray(entry) ? skuKey(entry[tuple.sku]) : null; + if (sku === null || was.dups.has(sku) || now.dups.has(sku) || page.dups.has(sku)) return false; + const state = now.map.get(sku); + const offer = page.map.get(sku); + if (state) { + if (!offer || skuVerdict(state, offer, tuple, ctx.vocabulary) !== true) return false; + const old = was.map.get(sku); + if (old && skuVerdict(old, offer, tuple, ctx.vocabulary) === true) return false; + } else if (offer) { + return false; + } + } + return true; + } catch { + return false; + } +}; + +/** + * The slot indices at which two signatures differ, or null when that cannot be said: either side + * is a status-signal literal (a state, not slots), the shapes differ, or — defensively — the + * strings differ with no slot differing. Null means "treat as a change of everything", the safe + * reading for every caller (nothing is ignored, nothing is caught up). + */ +export const changedSlots = (storedSignature, observedSignature) => { + const before = slotsOf(storedSignature); + const after = slotsOf(observedSignature); + if (!before || !after || before.length !== after.length) return null; + const changed = []; + for (let i = 0; i < after.length; i++) { + if (JSON.stringify(before[i]) !== JSON.stringify(after[i])) changed.push(i); + } + return changed.length ? changed : null; +}; + +/** A signature's slot values, or null for a literal (exported for the sweep's caught-up test). */ +export const signatureSlots = (signature) => slotsOf(signature); + +const factString = (value) => (typeof value === 'string' && value !== '' ? value : null); +const factNumber = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : null); + +/** + * The renderer's `pageFacts` reduced to the contract's shape, in a FIXED key order — so the stored + * string is canonical (equal facts store equal bytes) and a renderer that sends extra keys or a + * wrong type cannot put anything into the record the comparators do not expect. A record where + * every fact is empty is null: it can support no comparison, and null says so for free. + */ +export const canonicalPageFacts = (raw) => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null; + const p = raw.product; + const product = + p && typeof p === 'object' && !Array.isArray(p) + ? { + name: factString(p.name), + brand: factString(p.brand), + image: factString(p.image), + rating: Array.isArray(p.rating) ? [factNumber(p.rating[0]), factNumber(p.rating[1])] : null, + offers: Array.isArray(p.offers) + ? p.offers.map((offer) => + Array.isArray(offer) + ? [factString(offer[0]), factString(offer[1]), factString(offer[2]), factString(offer[3])] + : [null, null, null, null] + ) + : null, + } + : null; + const facts = { + canonical: factString(raw.canonical), + title: factString(raw.title), + metaDescription: factString(raw.metaDescription), + h1: factString(raw.h1), + product: + product && (product.name || product.brand || product.image || product.rating || product.offers) ? product : null, + breadcrumbs: + Array.isArray(raw.breadcrumbs) && raw.breadcrumbs.length && raw.breadcrumbs.every((crumb) => factString(crumb)) + ? raw.breadcrumbs.slice() + : null, + }; + return Object.values(facts).some((value) => value !== null) ? facts : null; +}; + +/** + * The page record as stored: `{ json, bytes, refused }`. `json` is the canonical string, or null + * when there is nothing to store OR the record is REFUSED for exceeding PAGE_FACTS_MAX_BYTES. + * + * WHY 16 KB. The record is written on every render of a mapped URL into a node-local table that + * holds one row per probed URL (hundreds of thousands per node), and it is read on every probe of + * that URL — so its size is paid in both places, at corpus scale. A typical product page's record + * is 1-3 KB (the renderer measures well under 2 KB; 50 SKU offers add ~2 KB); even the renderer's + * own 200-offer cap fits at ~10 KB. The renderer's worst case is ~125 KB, and a page that large is + * pathological: storing it would make one URL's row cost what fifty normal ones do. Refused rather + * than truncated, like the renderer's own bounds — a truncated offer list or trail would disagree + * with the endpoint forever — and refusal stores NULL, so that page simply has no record (no + * claim) until a render fits. + */ +export const serializePageFacts = (raw) => { + const facts = canonicalPageFacts(raw); + if (!facts) return { json: null, bytes: 0, refused: false }; + const json = JSON.stringify(facts); + const bytes = Buffer.byteLength(json, 'utf8'); + return bytes > PAGE_FACTS_MAX_BYTES ? { json: null, bytes, refused: true } : { json, bytes, refused: false }; +}; + +/** A stored page record parsed, or null for anything that is not one (absent, corrupted, legacy). */ +export const parsePageFacts = (json) => { + if (typeof json !== 'string' || json === '') return null; + try { + const facts = JSON.parse(json); + return facts && typeof facts === 'object' && !Array.isArray(facts) ? facts : null; + } catch { + return null; + } +}; + /** * The signature a declared status signal assigns to this response, or null when none applies. * @@ -629,13 +1273,54 @@ export const substituteTemplate = (template, match) => * value in any other extracted field. This is how a rule watches per-variant state without * signing the whole variant object, whose other fields (inventory counters, store data) move * without the page moving. + * + * A trailing TUPLE projection, `[*].{a,b.c}`, projects each element to the tuple `[a, b.c]` + * (dotted inner paths, no brackets, no nesting; an unreachable inner path is null). It exists so + * per-variant state can be compared BY KEY: `variants[*].{sku,availability,price.value}` keeps each + * SKU's availability and price attached to its SKU, where two separate `[*]` projections are each + * sorted on their own and lose the pairing. Tuples sort by their JSON like any projected element. */ -export const valueAtPath = (value, path) => walkPath(value, String(path).match(PATH_TOKEN) ?? [], 0); +export const valueAtPath = (value, path) => walkPath(value, tokensOf(path), 0); // `[*]` and `[N]` are tried BEFORE the bare-name alternative: `*` and digits are legal name // characters to that alternative, so ordering it first would tokenize `[*]` as the name `*`. const PATH_TOKEN = /\[\*\]|\[\d+\]|[^.[\]]+/g; +// `.{inner,inner.path}` — the only place a tuple may appear. +const TUPLE_TAIL = /^(.*\[\*\])\.\{([^{}]*)\}$/; +// One inner path of a tuple: dotted names, no brackets, braces, commas or empty segments. +const TUPLE_INNER = /^[^.[\]{},]+(?:\.[^.[\]{},]+)*$/; + +/** + * Why an extract path cannot be used, or null. Only tuple syntax is checked — any other string is a + * path, and a path to nowhere is data (it extracts null), not a config error. Braces anywhere but a + * well-formed trailing tuple, though, can only be a mistyped projection that would silently + * extract null on every probe. + */ +export const extractPathProblem = (path) => { + const text = String(path); + if (!text.includes('{') && !text.includes('}')) return null; + const match = TUPLE_TAIL.exec(text); + if (!match || match[1].includes('{') || match[1].includes('}')) { + return 'a tuple projection must END the path, directly after [*] — e.g. "items[*].{sku,price.value}"'; + } + if (match[2].split(',').some((inner) => !TUPLE_INNER.test(inner.trim()))) { + return 'each name inside {…} must be a dotted path (no brackets, no empty names)'; + } + return null; +}; + +// A path's tokens; a well-formed trailing tuple becomes one final `{ tuple: [tokens, ...] }` token. +const tokensOf = (path) => { + const text = String(path); + const match = extractPathProblem(text) === null ? TUPLE_TAIL.exec(text) : null; + if (!match) return text.match(PATH_TOKEN) ?? []; + return [ + ...(match[1].match(PATH_TOKEN) ?? []), + { tuple: match[2].split(',').map((inner) => inner.trim().match(PATH_TOKEN) ?? []) }, + ]; +}; + // A total, stable order over projected elements of any shape — they may be strings, numbers, nulls // or whole objects. Comparing their JSON keeps the sort deterministic across passes, which is the // only property that matters here: the same multiset must always produce the same signature. @@ -648,8 +1333,17 @@ const byProjectedValue = (a, b) => { const walkPath = (value, tokens, from) => { let current = value; for (let i = from; i < tokens.length; i++) { - if (current === null || current === undefined) return undefined; const token = tokens[i]; + if (typeof token === 'object') { + // The tuple: always the last token, and always reached through a `[*]`, so `current` is one + // element. An element that is not there still yields a tuple (of nulls), so every element of + // the projection has the same shape. + return token.tuple.map((inner) => { + const field = current === null || current === undefined ? undefined : walkPath(current, inner, 0); + return field === undefined ? null : field; + }); + } + if (current === null || current === undefined) return undefined; if (token === '[*]') { if (!Array.isArray(current)) return undefined; const projected = current.map((element) => { diff --git a/packages/plugin/test/changeProbe.test.js b/packages/plugin/test/changeProbe.test.js index 8a9b6829..7c55eae8 100644 --- a/packages/plugin/test/changeProbe.test.js +++ b/packages/plugin/test/changeProbe.test.js @@ -1059,22 +1059,25 @@ test('writeSignature: patch for an existing row (claim untouched unless cleared) assert.deepEqual(Object.keys(calls.patch[0].fields).sort(), ['probedAt', 'signature']); await changeProbe.writeSignature(URL_A, 'sig', { rowExists: true, clearClaim: true }); - // BOTH halves of the claim pair. `pageClaimAt` is the render `pageSignature` came from; clearing - // one and leaving the other is a half-state a verification could later read. + // THE WHOLE RECORD. `pageClaimAt` is the render `pageSignature` and `pageFacts` came from; clearing + // one and leaving another is a half-state a verification (or a mapped field) could later read. assert.deepEqual(Object.keys(calls.patch[1].fields).sort(), [ 'pageClaimAt', + 'pageFacts', 'pageSignature', 'probedAt', 'signature', ]); assert.equal(calls.patch[1].fields.pageSignature, null); assert.equal(calls.patch[1].fields.pageClaimAt, null); + assert.equal(calls.patch[1].fields.pageFacts, null, 'the page record describes the expired page too'); await changeProbe.writeSignature(URL_A, 'sig', { rowExists: false }); assert.equal(calls.patch.length, 2); assert.equal(calls.put.length, 1); assert.equal(calls.put[0].rowFields.url, URL_A); assert.equal(calls.put[0].rowFields.pageSignature, null, 'a created row starts with no claim'); + assert.equal(calls.put[0].rowFields.pageFacts, null, 'and with no page record'); }); // ---- continuous pacing + the local-load governor ------------------------------------------------ @@ -2004,3 +2007,784 @@ test('APPENDED path: a baseline without the shape the shorter rule writes re-bas assert.equal(stats.extended, 0); assert.deepEqual(triggered, []); }); + +// ---- the page record: mapped fields, caught-up changes, ignored slots, the mapping guard ----------- + +/** + * A rule that maps four of its six slots to page facts and ignores one: + * 0 title -> title (text) 1 seoUrl -> canonical (path) 2 image -> product.image (path) + * 3 skus[*].{sku,availability,price} -> product.offers (skus) + * 4 inventory — ignoreChanges 5 regular — unmapped, a change here always triggers + * Stored and observed signatures are built with the real extraction, and records with the real + * canonicalizer, so these exercise the strings production writes. + */ +const MAPPED_RULE = { + label: 'pdp', + pathPattern: '^/product/prd-([^/]+)', + source: 'request', + request: { urlTemplate: 'https://api.example.com/p/$1', method: 'POST', body: '{}' }, + extract: ['title', 'seoUrl', 'image', 'skus[*].{sku,availability,price}', 'inventory', 'regular'], + statusSignals: [{ status: 404, contains: 'GONE', signature: 'unavailable' }], + invalidateScope: 'route:prefix:/product/', + pageCheck: { + enabled: true, + fields: [ + { slot: 0, fact: 'title', compare: 'text' }, + { slot: 1, fact: 'canonical', compare: 'path' }, + { slot: 2, fact: 'product.image', compare: 'path' }, + { slot: 3, fact: 'product.offers', compare: 'skus' }, + ], + ignoreChanges: [4], + }, +}; +const API = (over = {}) => ({ + title: 'Red Shoe', + seoUrl: '/product/prd-a/red-shoe.jsp', + image: 'https://media.example.com/i/shoe?w=350', + skus: [ + { sku: '111', availability: 'In Stock', price: 19.99 }, + { sku: '222', availability: 'Out of Stock', price: null }, + ], + inventory: 12, + regular: 39.99, + ...over, +}); +const apiSig = async (over, raw = MAPPED_RULE) => signedBy(raw, API(over)); +const RECORD = async (over = {}) => { + const { canonicalPageFacts } = await import('../src/util/changeProbeSpec.js'); + return JSON.stringify( + canonicalPageFacts({ + canonical: 'https://www.example.com/product/prd-a/red-shoe.jsp', + title: 'Red Shoe', + product: { + image: 'https://media.example.com/i/shoe?w=1000', + offers: [ + ['111', '19.99', 'USD', 'InStock'], + ['222', null, 'USD', 'OutOfStock'], + ], + }, + ...over, + }) + ); +}; +// A WITNESSED record: rendered (2000) after the baseline was taken (1000). +const BASELINE_AT = 1000; +const RENDERED_AT = new Date(2000); + +const guardFor = (settings = { threshold: 0.2, minWitnessed: 10 }) => { + const disarmed = []; + const guard = changeProbe.createMappingGuard({ + settings: () => settings, + onDisarm: (rule, field, entry) => disarmed.push({ rule: rule.label, field: field.label, ...entry }), + }); + return { guard, disarmed }; +}; + +const runMappedPass = async ({ rulesRaw = [MAPPED_RULE], rows, answers, stored = {}, ...overrides }) => { + const { compileProbeRules } = await import('../src/util/changeProbeSpec.js'); + const { createInlineTrigger } = await import('../src/util/triggerQueue.js'); + const rules = compileProbeRules(rulesRaw); + const written = []; + const triggered = []; + const verified = []; + const write = async (url, signature, options = {}) => written.push({ url, signature, ...options }); + const triggers = createInlineTrigger({ trigger: async (target) => triggered.push(target.url), write }); + const stats = await changeProbe.runProbePass({ + rows: stream(rows), + rules, + ownerOf: () => 'node-a', + hostname: 'node-a', + probe: async (rule, url) => answers[url] ?? null, + // Steady state by default: the stored fingerprint is the rule's own, so nothing is stamped. + read: async (url) => + stored[url] + ? { + probedAt: BASELINE_AT, + pageSignature: null, + pageClaimAt: RENDERED_AT, + pageFacts: null, + fingerprint: rules[0].fingerprint, + ...stored[url], + } + : null, + write, + submitTrigger: triggers.submit, + verify: async (url, basisAt) => verified.push({ url, basisAt }), + isArmed: async () => false, + dryRun: false, + maxTriggers: 1000, + concurrency: 1, + ratePerSecond: 1000, + pause: async () => {}, + ...overrides, + }); + await triggers.drain(); + stats.triggered = triggers.stats.triggered; + assertInvariant(stats); + return { stats, written, triggered, verified, rules }; +}; + +test('MAPPED FIELD: the page disagrees with an UNCHANGED origin -> trigger, counted per field, record cleared', async () => { + const signature = await apiSig(); + const { stats, triggered, written } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Red Shoe (Old Name)' }) } }, + }); + assert.deepEqual(triggered, [URL_A]); + assert.equal(stats.unchanged, 1, 'the origin did not change — the bucket says so'); + assert.equal(stats.pageMismatch, 1); + assert.deepEqual(stats.fieldMismatch, { pdp: { '0:title': 1 } }, 'decomposed by field'); + assert.deepEqual(stats.slotChanges, {}, 'no origin change to decompose'); + assert.equal(written[0].clearClaim, true, 'the trigger clears the record with the claim'); +}); + +test('the same page AGREEING on every mapped field -> unchanged, nothing triggered, nothing written', async () => { + const signature = await apiSig(); + const { stats, triggered, written } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageFacts: await RECORD() } }, + }); + assert.deepEqual(triggered, []); + assert.deepEqual(written, [], 'a converged corpus still pays no write per probe'); + assert.equal(stats.pageMismatch, 0); + assert.deepEqual(stats.fieldMismatch, {}); +}); + +test('CAUGHT UP: the origin changed and the page ALREADY shows the new value -> baseline moves, nothing triggered', async () => { + // A cadence render landed after the rename: re-rendering again would buy nothing. + const before = await apiSig(); + const after = await apiSig({ title: 'Red Running Shoe' }); + const { stats, triggered, written } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: after }, + stored: { [URL_A]: { signature: before, pageFacts: await RECORD({ title: 'Red Running Shoe' }) } }, + }); + assert.deepEqual(triggered, []); + assert.equal(stats.changed, 1, 'still a change — for the canary, the rest of the corpus has not caught up'); + assert.equal(stats.caughtUp, 1); + assert.deepEqual(stats.slotChanges, { pdp: { 0: 1 } }); + assert.equal(written.length, 1); + assert.equal(written[0].signature, after, 'the new baseline, so the next pass compares against it'); + assert.equal(written[0].rowExists, true); + assert.notEqual(written[0].clearClaim, true, 'nothing was expired, so the record stands'); +}); + +test('a change the page has NOT caught up with triggers — and is a page mismatch too', async () => { + const { stats, triggered } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'Red Running Shoe' }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD() } }, + }); + assert.deepEqual(triggered, [URL_A]); + assert.equal(stats.caughtUp, 0); + assert.equal(stats.changed, 1); + assert.deepEqual(stats.fieldMismatch, { pdp: { '0:title': 1 } }); +}); + +test('a change in an UNMAPPED slot triggers exactly as before, even when every mapped field agrees', async () => { + const { stats, triggered } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ regular: 34.99 }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD() } }, + }); + assert.deepEqual(triggered, [URL_A]); + assert.equal(stats.caughtUp, 0); + assert.deepEqual(stats.slotChanges, { pdp: { 5: 1 } }); + // ...and a change spanning a caught-up slot AND an unmapped one still triggers. + const both = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'Red Running Shoe', regular: 34.99 }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD({ title: 'Red Running Shoe' }) } }, + }); + assert.deepEqual(both.triggered, [URL_A]); + assert.equal(both.stats.caughtUp, 0); +}); + +test('a change in a mapped slot with NO page record (or a record that cannot compare) triggers — no evidence, no suppression', async () => { + for (const pageFacts of [null, await RECORD({ title: null }), '{corrupt']) { + const { stats, triggered } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'Red Running Shoe' }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts } }, + }); + assert.deepEqual(triggered, [URL_A], `record ${pageFacts}`); + assert.equal(stats.caughtUp, 0); + assert.equal(stats.pageMismatch, 0, 'and no claim is not a mismatch'); + } +}); + +test('skus: a changed SKU the page does not list is NOT caught up, even though every listed SKU agrees', async () => { + const skus = (price) => [ + { sku: '111', availability: 'In Stock', price: 19.99 }, + { sku: '999', availability: 'In Stock', price }, // beyond the page's truncated offer list + ]; + const { triggered, stats } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ skus: skus(4) }) }, + stored: { [URL_A]: { signature: await apiSig({ skus: skus(5) }), pageFacts: await RECORD() } }, + }); + assert.deepEqual(triggered, [URL_A], 'swallowing this change is the failure caught-up must not have'); + assert.equal(stats.caughtUp, 0); + assert.equal(stats.pageMismatch, 0, 'the SKUs both sides list agree — the page is not wrong where it can be checked'); + // A repriced SKU the page DOES list, already showing the new price: caught up. + const listed = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig() }, + stored: { + [URL_A]: { + signature: await apiSig({ skus: [{ sku: '111', availability: 'In Stock', price: 21.99 }, API().skus[1]] }), + pageFacts: await RECORD(), + }, + }, + }); + assert.deepEqual(listed.triggered, []); + assert.equal(listed.stats.caughtUp, 1); +}); + +test('IGNORED: a change confined to ignoreChanges writes the baseline, triggers nothing, and is NOT a change for the canary', async () => { + const rows = []; + const answers = {}; + const stored = {}; + for (let i = 0; i < 10; i++) { + const url = `https://example.com/product/prd-${i}/`; + rows.push(row(url)); + answers[url] = await apiSig({ inventory: 11 - i }); + stored[url] = { signature: await apiSig(), pageFacts: null }; + } + const { stats, triggered, written } = await runMappedPass({ rows, answers, stored }); + assert.deepEqual(triggered, []); + assert.equal(stats.ignored, 10); + assert.equal(stats.unchanged, 10, 'bucketed unchanged: for mass-change detection it is not a change'); + assert.equal(stats.changed, 0); + assert.deepEqual(stats.slotChanges, { pdp: { 4: 10 } }, 'but still counted per slot'); + assert.equal(written.length, 10, 'each baseline moves, so the next pass compares against the new value'); + // THE CANARY: a site-wide edit of an ignored field must not trip a route-wide invalidation. + assert.equal(changeProbe.canaryVerdict(stats, { threshold: 0.1, minSample: 5 }).tripped, false); + // Control: the same volume of change in a slot that is NOT ignored trips it. + for (const url of Object.keys(answers)) answers[url] = await apiSig({ regular: 1 }); + const control = await runMappedPass({ rows, answers, stored }); + assert.equal(control.stats.changed, 10); + assert.equal(changeProbe.canaryVerdict(control.stats, { threshold: 0.1, minSample: 5 }).tripped, true); +}); + +test('ignored + caught up together: nothing triggers; ignored + unmapped: triggers', async () => { + const quiet = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'Red Running Shoe', inventory: 3 }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD({ title: 'Red Running Shoe' }) } }, + }); + assert.deepEqual(quiet.triggered, []); + assert.equal(quiet.stats.caughtUp, 1); + assert.equal(quiet.stats.ignored, 0, 'not ignored-only: the title changed too'); + const loud = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ regular: 1, inventory: 3 }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD() } }, + }); + assert.deepEqual(loud.triggered, [URL_A]); +}); + +test('a status-signal literal transition is never ignored or caught up — it is a state, not slots', async () => { + const { stats, triggered } = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: 'unavailable' }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD() } }, + }); + assert.deepEqual(triggered, [URL_A]); + assert.deepEqual(stats.slotChanges, { pdp: { signal: 1 } }); + assert.equal(stats.pageMismatch, 0, 'a literal carries no values to compare'); +}); + +test('APPENDED path (#206) rows: a mapped disagreement triggers and a caught-up change is absorbed and upgraded', async () => { + const { compileProbeRules } = await import('../src/util/changeProbeSpec.js'); + const OLD = { ...MAPPED_RULE, extract: MAPPED_RULE.extract.slice(0, 5), pageCheck: undefined }; + const oldFp = compileProbeRules([OLD])[0].fingerprint; + const { guard } = guardFor({ threshold: 0, minWitnessed: 1 }); // disarms on its first witness + // Unchanged prefix, page wrong on a mapped field -> trigger. + const wrong = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig() }, + stored: { + [URL_A]: { signature: await apiSig({}, OLD), fingerprint: oldFp, pageFacts: await RECORD({ title: 'Old' }) }, + }, + guard, + }); + assert.equal(wrong.stats.extended, 1); + assert.deepEqual(wrong.triggered, [URL_A]); + // An appended-path row is never WITNESSED: its new slots have no baseline to be unchanged against. + assert.equal(wrong.stats.fieldGuard.pdp['0:title'].witnessed, 0); + // Changed old slot the page already shows -> caught up, and the write upgrades the baseline. + const caught = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'Red Running Shoe' }) }, + stored: { + [URL_A]: { + signature: await apiSig({}, OLD), + fingerprint: oldFp, + pageFacts: await RECORD({ title: 'Red Running Shoe' }), + }, + }, + }); + assert.equal(caught.stats.extended, 1); + assert.equal(caught.stats.caughtUp, 1); + assert.deepEqual(caught.triggered, []); + assert.equal(caught.written[0].signature, await apiSig({ title: 'Red Running Shoe' }), 'the full observation'); + assert.equal(caught.written[0].fingerprint, caught.rules[0].fingerprint, 'under the current fingerprint'); +}); + +test('MAPPING GUARD: a broken mapping is DISARMED after minWitnessed and stops triggering; it still counts', async () => { + const { guard, disarmed } = guardFor({ threshold: 0.2, minWitnessed: 10 }); + const rows = []; + const answers = {}; + const stored = {}; + const signature = await apiSig(); + for (let i = 0; i < 30; i++) { + const url = `https://example.com/product/prd-${i}/`; + rows.push(row(url)); + answers[url] = signature; + // Every page "disagrees" on the title: the slot is mapped to the wrong fact. + stored[url] = { signature, pageFacts: await RECORD({ title: 'Acme' }) }; + } + const { stats, triggered } = await runMappedPass({ rows, answers, stored, guard }); + // Witness 10 crosses the threshold and is itself not acted on; 1-9 each re-rendered a page. + assert.equal(triggered.length, 9, 'a corpus-wide broken mapping costs ~minWitnessed renders, not the corpus'); + assert.equal(stats.pageMismatch, 9, 'pageMismatch counts only the actionable (armed) disagreements'); + assert.deepEqual(stats.fieldMismatch, { pdp: { '0:title': 30 } }, 'a disarmed field is still counted'); + assert.deepEqual( + disarmed.map((d) => [d.rule, d.field, d.witnessed, d.disagreed]), + [['pdp', '0:title', 10, 10]] + ); + assert.deepEqual(stats.fieldGuard.pdp['0:title'], { witnessed: 30, disagreed: 30, armed: false }); + assert.deepEqual( + stats.fieldGuard.pdp['1:canonical'], + { witnessed: 30, disagreed: 0, armed: true }, + 'siblings unaffected' + ); +}); + +test('MAPPING GUARD: rare round trips never disarm — and each one re-renders', async () => { + const { guard, disarmed } = guardFor({ threshold: 0.2, minWitnessed: 20 }); + const rows = []; + const answers = {}; + const stored = {}; + const signature = await apiSig(); + for (let i = 0; i < 100; i++) { + const url = `https://example.com/product/prd-${i}/`; + rows.push(row(url)); + answers[url] = signature; + // 2% of pages caught a transient value mid-flip: the genuine round-trip case. + stored[url] = { signature, pageFacts: await RECORD(i % 50 === 7 ? { title: 'Red Shoe — Sale' } : {}) }; + } + const { stats, triggered } = await runMappedPass({ rows, answers, stored, guard }); + assert.equal(triggered.length, 2, 'an individual witnessed disagreement is NEVER suppressed'); + assert.deepEqual(disarmed, []); + assert.deepEqual(stats.fieldGuard.pdp['0:title'], { witnessed: 100, disagreed: 2, armed: true }); +}); + +test('MAPPING GUARD: the rate is RECENT — a mapping right for a long time and then broken is still disarmed promptly', async () => { + const { guard, disarmed } = guardFor({ threshold: 0.2, minWitnessed: 10 }); // memory ~100 comparisons + const rule = { label: 'pdp', extract: ['title'] }; + const title = { slot: 0, fact: 'title', compare: 'text', options: {}, label: '0:title' }; + // A week of agreement... + for (let i = 0; i < 100_000; i++) guard.witness(rule, title, false); + assert.equal(guard.isArmed(rule, title), true); + // ...then the site changes its title template and every re-render disagrees. + let spent = 0; + while (guard.isArmed(rule, title) && spent < 1000) { + guard.witness(rule, title, true); + spent++; + } + assert.equal(disarmed.length, 1); + assert.ok(spent < 60, `disarmed after ${spent} disagreements, not after outweighing 100,000 agreements`); + // The halved sample never drops below minWitnessed, and a disarm is sticky. + for (let i = 0; i < 1000; i++) guard.witness(rule, title, false); + assert.equal(guard.isArmed(rule, title), false, 'only a mapping edit or a restart re-arms'); + // A correct mapping at a steady 2% never trips however long it runs. + const { guard: steady } = guardFor({ threshold: 0.2, minWitnessed: 10 }); + for (let i = 0; i < 100_000; i++) steady.witness(rule, title, i % 50 === 0); + assert.equal(steady.isArmed(rule, title), true); +}); + +test('MAPPING GUARD: only WITNESSED comparisons count — a record older than the baseline proves nothing', async () => { + const { guard, disarmed } = guardFor({ threshold: 0.2, minWitnessed: 10 }); + const rows = []; + const answers = {}; + const stored = {}; + const signature = await apiSig(); + for (let i = 0; i < 30; i++) { + const url = `https://example.com/product/prd-${i}/`; + rows.push(row(url)); + answers[url] = signature; + // Rendered BEFORE the baseline was taken: the page may simply predate the current value. + stored[url] = { signature, pageFacts: await RECORD({ title: 'Acme' }), pageClaimAt: new Date(500) }; + } + const { triggered, stats } = await runMappedPass({ rows, answers, stored, guard }); + assert.equal(triggered.length, 30); + assert.deepEqual(disarmed, []); + assert.equal(stats.fieldGuard.pdp['0:title'].witnessed, 0); + // A CHANGED origin is not witnessed either (the page cannot be expected to show a value it predates). + const changed = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'New' }) }, + stored: { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Acme' }) } }, + guard, + }); + assert.equal(changed.stats.fieldGuard.pdp['0:title'].witnessed, 0); +}); + +test('MAPPING GUARD: a disarmed field no longer vouches for a caught-up change — the change triggers', async () => { + const { guard } = guardFor({ threshold: 0.2, minWitnessed: 1 }); + const signature = await apiSig(); + await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Acme' }) } }, + guard, + }); + const { triggered, stats } = await runMappedPass({ + rows: [row(URL_B)], + answers: { [URL_B]: await apiSig({ title: 'Red Running Shoe' }) }, + stored: { [URL_B]: { signature, pageFacts: await RECORD({ title: 'Red Running Shoe' }) } }, + guard, + }); + assert.deepEqual(triggered, [URL_B]); + assert.equal(stats.caughtUp, 0); +}); + +test('MAPPING GUARD: stays disarmed across a reload of the SAME mapping, re-arms when the mapping changes', async () => { + const { guard } = guardFor({ threshold: 0.2, minWitnessed: 1 }); + const signature = await apiSig(); + const broken = { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Acme' }) } }; + await runMappedPass({ rows: [row(URL_A)], answers: { [URL_A]: signature }, stored: broken, guard }); + // A config reload compiles NEW field objects for the same mapping: still disarmed. + const reloaded = await runMappedPass({ rows: [row(URL_A)], answers: { [URL_A]: signature }, stored: broken, guard }); + assert.equal(reloaded.stats.fieldGuard.pdp['0:title'].armed, false); + assert.deepEqual(reloaded.triggered, []); + // The operator fixes the mapping (slot 0 now compares against h1): a new key, a clean record. + const fixed = { + ...MAPPED_RULE, + pageCheck: { + ...MAPPED_RULE.pageCheck, + fields: [{ slot: 0, fact: 'h1', compare: 'text' }, ...MAPPED_RULE.pageCheck.fields.slice(1)], + }, + }; + const after = await runMappedPass({ + rulesRaw: [fixed], + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Acme', h1: 'Red Shoe' }) } }, + guard, + }); + assert.equal(after.stats.fieldGuard.pdp['0:h1'].armed, true); + assert.deepEqual(after.triggered, []); +}); + +test('MAPPING GUARD in DRY RUN: counts and disarms (the dry-run week finds a broken mapping), triggers nothing', async () => { + const { guard, disarmed } = guardFor({ threshold: 0.2, minWitnessed: 5 }); + const rows = []; + const answers = {}; + const stored = {}; + const signature = await apiSig(); + for (let i = 0; i < 8; i++) { + const url = `https://example.com/product/prd-${i}/`; + rows.push(row(url)); + answers[url] = signature; + stored[url] = { signature, pageFacts: await RECORD({ title: 'Acme' }) }; + } + const { triggered, written } = await runMappedPass({ rows, answers, stored, guard, dryRun: true }); + assert.deepEqual(triggered, []); + assert.equal(disarmed.length, 1); + assert.ok( + written.every((w) => w.clearClaim !== true), + 'dry run never clears the record' + ); +}); + +test('the process guard WARNS loudly on disarm, naming the rule, the field and its extract path', async (t) => { + const { applyOptions } = await import('../src/config.js'); + t.after(() => applyOptions({})); + applyOptions({ changeProbe: { mappingGuard: { threshold: 0.5, minWitnessed: 2 } } }); + const warns = []; + globalThis.logger.warn = (message) => warns.push(message); + const { compileProbeRules } = await import('../src/util/changeProbeSpec.js'); + const [rule] = compileProbeRules([MAPPED_RULE]); + // The same factory the sweep uses, with the live config and the real warning. + const signature = await apiSig(); + const guard = changeProbe.__mappingGuardForTest(); + for (const url of [URL_A, URL_B]) { + await changeProbe.runProbePass({ + rows: stream([row(url)]), + rules: [rule], + ownerOf: () => 'node-a', + hostname: 'node-a', + probe: async () => signature, + read: async () => ({ + signature, + probedAt: BASELINE_AT, + pageClaimAt: RENDERED_AT, + pageFacts: await RECORD({ title: 'Acme' }), + fingerprint: rule.fingerprint, + }), + write: async () => {}, + submitTrigger: async () => 'queued', + guard, + dryRun: true, + maxTriggers: 10, + concurrency: 1, + ratePerSecond: 1000, + pause: async () => {}, + }); + } + assert.equal(warns.length, 1); + assert.match( + warns[0], + /change-probe pdp: pageCheck field 0:title \(extract\[0\] "title", compare text\) is DISARMED/ + ); + assert.match(warns[0], /2 of 2 recent witnessed comparisons \(100\.0%\)/); +}); + +test('verification: a pair+fields rule verifies only when NO armed mapped field disagrees', async () => { + const PAIRED = { + ...MAPPED_RULE, + extract: [...MAPPED_RULE.extract, 'price', 'available'], + pageCheck: { ...MAPPED_RULE.pageCheck, priceFrom: 6, availableFrom: 7 }, + }; + const json = { ...API(), price: 19.99, available: true }; + const signature = await signedBy(PAIRED, json); + const claim = JSON.stringify([['19.99'], true]); + const armedPass = (pageFacts) => + runMappedPass({ + rulesRaw: [PAIRED], + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageSignature: claim, pageFacts } }, + isArmed: async () => true, + }); + const agreeing = await armedPass(await RECORD()); + assert.deepEqual( + agreeing.verified.map((v) => v.url), + [URL_A] + ); + const wrong = await armedPass(await RECORD({ title: 'Acme' })); + assert.deepEqual(wrong.verified, [], 'a page wrong on a mapped field is never certified'); + assert.deepEqual(wrong.triggered, [URL_A]); + // No record (older renderer): the claim pair alone certifies, exactly as before fields existed. + const noRecord = await armedPass(null); + assert.deepEqual( + noRecord.verified.map((v) => v.url), + [URL_A] + ); +}); + +test('verification: a fields-only rule needs an armed mapped field that AGREED — no record, no proof', async () => { + const signature = await apiSig(); + const pass = (storedRow, answer = signature) => + runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: answer }, + stored: { [URL_A]: storedRow }, + isArmed: async () => true, + }); + assert.deepEqual( + (await pass({ signature, pageFacts: await RECORD() })).verified.map((v) => v.url), + [URL_A] + ); + assert.deepEqual((await pass({ signature, pageFacts: null })).verified, [], 'nothing was compared'); + // An ignored change: the page's visible fields did not move, and they agree — verified. + const ignored = await pass({ signature, pageFacts: await RECORD() }, await apiSig({ inventory: 1 })); + assert.equal(ignored.stats.ignored, 1); + assert.deepEqual( + ignored.verified.map((v) => v.url), + [URL_A] + ); + // A caught-up change: the baseline just moved — verified on the next pass, not this one. + const caught = await pass({ signature, pageFacts: await RECORD({ title: 'New' }) }, await apiSig({ title: 'New' })); + assert.equal(caught.stats.caughtUp, 1); + assert.deepEqual(caught.verified, []); +}); + +test('dry run: a caught-up change still moves the baseline; a mapped mismatch writes without clearing the record', async () => { + const caught = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: await apiSig({ title: 'New' }) }, + stored: { [URL_A]: { signature: await apiSig(), pageFacts: await RECORD({ title: 'New' }) } }, + dryRun: true, + }); + assert.equal(caught.written.length, 1); + assert.equal(caught.stats.caughtUp, 1); + const signature = await apiSig(); + const wrong = await runMappedPass({ + rows: [row(URL_A)], + answers: { [URL_A]: signature }, + stored: { [URL_A]: { signature, pageFacts: await RECORD({ title: 'Acme' }) } }, + dryRun: true, + }); + assert.deepEqual(wrong.triggered, []); + assert.equal(wrong.stats.pageMismatch, 1); + assert.equal(wrong.written.length, 1); + assert.notEqual(wrong.written[0].clearClaim, true); +}); + +test('slotChanges decomposes changed by slot for EVERY rule — a rule with no pageCheck too', async () => { + const { stats } = await runPass({ + rows: [row(URL_A), row(URL_B)], + answers: { [URL_A]: JSON.stringify([2]), [URL_B]: JSON.stringify([1]) }, + stored: { [URL_A]: JSON.stringify([1]), [URL_B]: JSON.stringify([1]) }, + }); + assert.equal(stats.changed, 1); + assert.deepEqual(stats.slotChanges, { pdp: { 0: 1 } }); + assert.equal(stats.fieldGuard, undefined, 'no mapped fields, no guard snapshot'); +}); + +// ---- recordPageClaim: the page record ------------------------------------------------------------- + +const mappedClaimHarness = async ({ existing = null, pageCheck } = {}) => { + const { applyOptions } = await import('../src/config.js'); + applyOptions({ + changeProbe: { + enabled: true, + rules: [ + { + label: 'pdp', + pathPattern: '^/product/prd-', + source: 'request', + request: { urlTemplate: 'https://api.example.com/x', method: 'POST', body: '{}' }, + extract: ['title', 'b', 'price', 'available'], + pageCheck: pageCheck ?? { + enabled: true, + priceFrom: 2, + availableFrom: 3, + fields: [{ slot: 0, fact: 'title', compare: 'text' }], + }, + }, + ], + }, + }); + const calls = { get: [], put: [], patch: [] }; + globalThis.databases.probe_state.ProbeState = { + async get(query) { + calls.get.push(query); + return existing; + }, + async put(id, row) { + calls.put.push({ id, row }); + }, + async patch(id, patch) { + calls.patch.push({ id, patch }); + }, + }; + return calls; +}; +const FACTS_IN = { title: 'Red Shoe', h1: 'Red Shoe', product: { offers: [['1', '35.99', 'USD', 'InStock']] } }; + +test('recordPageClaim stores the canonical page record in the SAME write as the claim and its stamp', async (t) => { + t.after(restoreConfig); + const calls = await mappedClaimHarness({ existing: { url: CLAIM_URL } }); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000, { pageFacts: FACTS_IN }); + const { canonicalPageFacts } = await import('../src/util/changeProbeSpec.js'); + assert.equal(calls.put.length, 0); + assert.equal(calls.patch.length, 1, 'one write — no extra read or write for the record'); + assert.deepEqual(calls.patch[0].patch, { + pageSignature: JSON.stringify([['35.99'], true]), + pageFacts: JSON.stringify(canonicalPageFacts(FACTS_IN)), + pageClaimAt: new Date(1_700_000_000_000), + }); + // A missing row is seeded with put, record included. + const seeded = await mappedClaimHarness(); + await changeProbe.recordPageClaim(CLAIM_URL, null, 1_700_000_000_000, { pageFacts: FACTS_IN }); + assert.equal(seeded.put.length, 1); + assert.equal(seeded.put[0].row.pageFacts, JSON.stringify(canonicalPageFacts(FACTS_IN))); + assert.equal( + seeded.put[0].row.pageSignature, + null, + 'the page declared no offers: no claim, and the record still stands' + ); +}); + +test('recordPageClaim: ABSENT pageFacts is an old renderer — warn hourly, and the record is stored EMPTY', async (t) => { + t.after(restoreConfig); + const calls = await mappedClaimHarness({ existing: { url: CLAIM_URL } }); + const warns = []; + globalThis.logger.warn = (message) => warns.push(message); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000); + assert.equal(warns.length, 1, 'throttled'); + assert.match(warns[0], /older than @harperfast\/prerender-browser 1\.37\.0/); + // Null, not left alone: the row's older record describes a page this render just replaced. + assert.equal(calls.patch[0].patch.pageFacts, null); + assert.equal(calls.patch[0].patch.pageSignature, JSON.stringify([['35.99'], true]), 'the claim is recorded as ever'); + // On a missing row with nothing to record, no row is created at all. + const empty = await mappedClaimHarness(); + await changeProbe.recordPageClaim(CLAIM_URL, null, 1_700_000_000_000); + assert.equal(empty.put.length + empty.patch.length, 0); +}); + +test('recordPageClaim: NULL pageFacts means the extraction ran and found nothing — silent, stored null', async (t) => { + t.after(restoreConfig); + const calls = await mappedClaimHarness({ existing: { url: CLAIM_URL } }); + const warns = []; + globalThis.logger.warn = (message) => warns.push(message); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000, { pageFacts: null }); + assert.equal(warns.length, 0); + assert.equal(calls.patch[0].patch.pageFacts, null); +}); + +test('recordPageClaim REFUSES an oversized record (stored null, warned), never truncates it', async (t) => { + t.after(restoreConfig); + const calls = await mappedClaimHarness({ existing: { url: CLAIM_URL } }); + const warns = []; + globalThis.logger.warn = (message) => warns.push(message); + const huge = { title: 'x', breadcrumbs: Array.from({ length: 30 }, () => 'y'.repeat(2000)) }; + await changeProbe.recordPageClaim(CLAIM_URL, null, 1_700_000_000_000, { pageFacts: huge }); + assert.equal(calls.patch[0].patch.pageFacts, null); + assert.match(warns[0], /over the 16384-byte bound — not stored/); +}); + +test('recordPageClaim: a render that did not store EVERY default device records no facts', async (t) => { + t.after(restoreConfig); + const calls = await mappedClaimHarness({ existing: { url: CLAIM_URL } }); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000, { + pageFacts: FACTS_IN, + complete: false, + }); + assert.equal(calls.patch[0].patch.pageFacts, null, 'the other device page is older than this record would claim'); +}); + +test('recordPageClaim: a fields-only rule records the page record and no claim; a pair-only rule no record', async (t) => { + t.after(restoreConfig); + const warns = []; + const fieldsOnly = await mappedClaimHarness({ + existing: { url: CLAIM_URL }, + pageCheck: { enabled: true, fields: [{ slot: 0, fact: 'title', compare: 'text' }] }, + }); + globalThis.logger.warn = (message) => warns.push(message); + await changeProbe.recordPageClaim(CLAIM_URL, undefined, 1_700_000_000_000, { pageFacts: FACTS_IN }); + assert.equal(fieldsOnly.patch[0].patch.pageSignature, null); + assert.ok(fieldsOnly.patch[0].patch.pageFacts); + assert.deepEqual(warns, [], 'no claim pair, so an old structuredOffers-less renderer is not its concern'); + // A rule with only the claim pair behaves exactly as before: no record, and no write at all + // when the render yields no claim. + const pairOnly = await mappedClaimHarness({ + existing: { url: CLAIM_URL }, + pageCheck: { enabled: true, priceFrom: 2, availableFrom: 3 }, + }); + await changeProbe.recordPageClaim(CLAIM_URL, ['35.99', 'USD', 'InStock'], 1_700_000_000_000, { pageFacts: FACTS_IN }); + assert.deepEqual(Object.keys(pairOnly.patch[0].patch).sort(), ['pageClaimAt', 'pageSignature']); + await changeProbe.recordPageClaim(CLAIM_URL, null, 1_700_000_000_000, { pageFacts: FACTS_IN }); + assert.equal(pairOnly.patch.length, 1, 'no claim, no write — exactly as before fields existed'); +}); + +test('changeProbeStatus names a rule’s mapped fields and ignored slots only when it has them', async (t) => { + t.after(restoreConfig); + await mappedClaimHarness({ + pageCheck: { enabled: true, fields: [{ slot: 0, fact: 'title', compare: 'text' }], ignoreChanges: [1] }, + }); + const status = await changeProbe.changeProbeStatus(); + assert.deepEqual(status.rules[0].pageFields, ['0:title:text']); + assert.deepEqual(status.rules[0].ignoreChanges, [1]); + await mappedClaimHarness({ pageCheck: { enabled: true, priceFrom: 2, availableFrom: 3 } }); + const plain = await changeProbe.changeProbeStatus(); + assert.deepEqual(Object.keys(plain.rules[0]).sort(), ['invalidateScope', 'label', 'pathPattern', 'source']); +}); diff --git a/packages/plugin/test/changeProbeSpec.test.js b/packages/plugin/test/changeProbeSpec.test.js index c6592584..41280002 100644 --- a/packages/plugin/test/changeProbeSpec.test.js +++ b/packages/plugin/test/changeProbeSpec.test.js @@ -31,6 +31,14 @@ import { prefixFingerprints, signatureUnderPrefix, availabilityToken, + extractPathProblem, + compareField, + fieldCaughtUp, + changedSlots, + canonicalPageFacts, + serializePageFacts, + parsePageFacts, + PAGE_FACTS_MAX_BYTES, } from '../src/util/changeProbeSpec.js'; const REQUEST_RULE = { @@ -300,7 +308,13 @@ test('pageCheck compiles only with in-bounds indices, and only for source "reque extract: ['a', 'b', 'c', 'd'], }; const ok = compileProbeRules([{ ...base, pageCheck: { enabled: true, priceFrom: 2, availableFrom: 3 } }]); - assert.deepEqual(ok[0].pageCheck, { priceFrom: 2, availableFrom: 3, vocabulary: null }); + assert.deepEqual(ok[0].pageCheck, { + priceFrom: 2, + availableFrom: 3, + vocabulary: null, + fields: [], + ignoreChanges: [], + }); // out of bounds -> dropped whole, rule survives (a half-applied mapping compares the wrong column) const oob = compileProbeRules([{ ...base, pageCheck: { enabled: true, priceFrom: 2, availableFrom: 9 } }]); @@ -506,7 +520,13 @@ test('pageCheck.availableValues / unavailableValues compile into the vocabulary, ], warnings ); - assert.deepEqual(clash[0].pageCheck, { priceFrom: 0, availableFrom: 1, vocabulary: null }); + assert.deepEqual(clash[0].pageCheck, { + priceFrom: 0, + availableFrom: 1, + vocabulary: null, + fields: [], + ignoreChanges: [], + }); assert.match(warnings[0], /both list x/); // A malformed list drops the vocabulary, not the page check. const bad = compileProbeRules( @@ -771,3 +791,777 @@ test('claimsDisagree survives a corrupted stored claim instead of ending the swe assert.equal(claimsDisagree(good, junk), false, `api claim ${junk} must not throw or disagree`); } }); + +// ---- tuple projection --------------------------------------------------------------------------- + +test('a trailing [*].{a,b.c} projects each element to a tuple, sorted like any projection', () => { + const doc = { + payload: { + products: [ + { + variants: [ + { sku: '222', availability: 'Out of Stock', price: { current: null } }, + { sku: '111', availability: 'In Stock', price: { current: 19.99 } }, + { sku: '333', price: {} }, + null, + ], + }, + ], + }, + }; + assert.deepEqual(valueAtPath(doc, 'payload.products[0].variants[*].{sku,availability,price.current}'), [ + // Sorted by each tuple's JSON: `["111",…]` < `["222",…]` < `["333",…]` < `[null,…]`. + ['111', 'In Stock', 19.99], + ['222', 'Out of Stock', null], + ['333', null, null], // an unreachable inner path is null, never a shifted neighbour + [null, null, null], // a null ELEMENT still yields a tuple, so every element has one shape + ]); + // Whitespace around the names is not part of them. + assert.deepEqual(valueAtPath({ v: [{ a: 1, b: { c: 2 } }] }, 'v[*].{ a , b.c }'), [[1, 2]]); + // Nested projections still work, the tuple applying to the innermost elements. + assert.deepEqual( + valueAtPath( + { + o: [ + { + i: [ + { x: 2, y: 'b' }, + { x: 1, y: 'a' }, + ], + }, + ], + }, + 'o[*].i[*].{x,y}' + ), + [ + [ + [1, 'a'], + [2, 'b'], + ], + ] + ); + // Not an array / missing branch: unreachable, like any other path. + assert.equal(valueAtPath({ v: 'str' }, 'v[*].{a}'), undefined); + assert.equal(valueAtPath({}, 'v[*].{a}'), undefined); +}); + +test('a reordered variant list projects to the same tuples, and a moved price reads as a change', () => { + const one = { + v: [ + { s: 'a', p: 1 }, + { s: 'b', p: 2 }, + ], + }; + const two = { + v: [ + { s: 'b', p: 2 }, + { s: 'a', p: 1 }, + ], + }; + const moved = { + v: [ + { s: 'a', p: 2 }, + { s: 'b', p: 1 }, + ], + }; + const sign = (json) => signatureOf(extractValues(json, ['v[*].{s,p}'])); + assert.equal(sign(one), sign(two), 'order is not a change'); + // The case two separate `[*]` projections cannot see: both prices swapped between SKUs. The sets + // of SKUs and of prices are unchanged, but the PAIRING moved — which is what a page shows. + assert.equal( + signatureOf(extractValues(one, ['v[*].s', 'v[*].p'])), + signatureOf(extractValues(moved, ['v[*].s', 'v[*].p'])) + ); + assert.notEqual(sign(one), sign(moved), 'a tuple keeps each price attached to its SKU'); +}); + +test('extractPathProblem refuses braces anywhere but a well-formed trailing tuple after [*]', () => { + for (const ok of ['a.b', 'a[*].b', 'a[*].{b}', 'a[*].{b,c.d}', 'x[0].y[*].{ s , p.v }', 'a[*].b[*].{c}']) { + assert.equal(extractPathProblem(ok), null, ok); + } + for (const bad of [ + 'a.{b,c}', // not after [*] + 'a[*].{b,c}.d', // not at the end + 'a[*].{b[0]}', // brackets inside + 'a[*].{b,,c}', // empty name + 'a[*].{}', // empty tuple + 'a[*].{b.}', // empty segment + 'a[*].{b', // unclosed + 'a[*].{x}[*].{y}', // two tuples + 'a[*].{b{c}}', // nested + ]) { + assert.notEqual(extractPathProblem(bad), null, bad); + } +}); + +test('a rule with a malformed tuple projection is dropped with a warning naming the path', () => { + const warnings = []; + const rules = compileProbeRules( + [{ ...REQUEST_RULE, label: 'bad', extract: ['a', 'v.{s,p}'] }, REQUEST_RULE], + warnings + ); + assert.equal(rules.length, 1, 'only the malformed rule drops'); + assert.match(warnings[0], /bad: extract\[1\] "v\.\{s,p\}": a tuple projection must END the path/); +}); + +test('the tuple path is part of the rule fingerprint — a different projection is a different observation', () => { + const fp = (extract) => compileProbeRules([{ ...REQUEST_RULE, extract }])[0].fingerprint; + assert.notEqual(fp(['v[*].{s,p}']), fp(['v[*].{s}'])); + assert.notEqual(fp(['v[*].{s,p}']), fp(['v[*].s'])); + assert.equal(fp(['v[*].{s,p}']), fp(['v[*].{s,p}'])); +}); + +// ---- pageCheck.fields / ignoreChanges compilation ------------------------------------------------- + +const MAPPED_BASE = { + label: 'm', + pathPattern: '^/p/', + source: 'request', + request: { urlTemplate: 'https://api.example.com/x', method: 'POST', body: '{}' }, + extract: ['title', 'seo', 'image', 'price', 'skus[*].{sku,availability,price}', 'crumbs', 'rating', 'inventory'], +}; +const mapped = (pageCheck, warnings = []) => + compileProbeRules([{ ...MAPPED_BASE, pageCheck: { enabled: true, ...pageCheck } }], warnings)[0]; + +test('pageCheck.fields compile into { slot, fact, compare, options, label }; the claim pair becomes optional', () => { + const warnings = []; + const rule = mapped( + { + fields: [ + { slot: 0, fact: 'title', compare: 'text' }, + { slot: 1, fact: 'canonical', compare: 'path' }, + { slot: 2, fact: 'product.image', compare: 'path' }, + { slot: 3, fact: 'product.offers', compare: 'priceSet' }, + { slot: 4, fact: 'product.offers', compare: 'skus' }, + { slot: 5, fact: 'breadcrumbs', compare: 'names', nameKey: 'label' }, + { slot: 6, fact: 'product.rating.value', compare: 'number', tolerance: 0.05 }, + { slot: 0, fact: 'h1', compare: 'text' }, // one slot, two facts: two fields + ], + ignoreChanges: [7, 7], + }, + warnings + ); + assert.deepEqual(warnings, []); + assert.equal(rule.pageCheck.priceFrom, null, 'no claim pair given, none compiled — and no warning'); + assert.equal(rule.pageCheck.availableFrom, null); + assert.deepEqual( + rule.pageCheck.fields.map((field) => field.label), + [ + '0:title', + '1:canonical', + '2:product.image', + '3:product.offers', + '4:product.offers', + '5:breadcrumbs', + '6:product.rating.value', + '0:h1', + ] + ); + assert.deepEqual(rule.pageCheck.fields[4].options, { tuple: { sku: 0, availability: 1, price: 2 } }); + assert.deepEqual(rule.pageCheck.fields[5].options, { nameKey: 'label' }); + assert.deepEqual(rule.pageCheck.fields[6].options, { tolerance: 0.05 }); + assert.deepEqual(rule.pageCheck.ignoreChanges, [7], 'de-duplicated'); + // The claim pair still compiles beside fields, exactly as before. + const both = mapped({ priceFrom: 3, availableFrom: 4, fields: [{ slot: 0, fact: 'title', compare: 'text' }] }); + assert.equal(both.pageCheck.priceFrom, 3); + assert.equal(both.pageCheck.fields.length, 1); +}); + +test('a bad pageCheck.fields entry drops ALONE with a warning — never the rule, never its siblings', () => { + const cases = [ + [{ slot: 99, fact: 'title', compare: 'text' }, /fields\[0\]\.slot must be an integer index into extract \(0-7\)/], + [{ slot: '0', fact: 'title', compare: 'text' }, /slot must be an integer/], + [{ slot: 0, fact: 'titel', compare: 'text' }, /fact must be one of canonical, title/], + [{ slot: 0, fact: 'title', compare: 'fuzzy' }, /compare must be one of text, path/], + [{ slot: 0, fact: 'title', compare: 'priceSet' }, /compare "priceSet" does not apply to fact "title" \(use text\)/], + [ + { slot: 3, fact: 'product.offers', compare: 'text' }, + /does not apply to fact "product.offers" \(use priceSet or skus\)/, + ], + [{ slot: 1, fact: 'canonical', compare: 'number' }, /use text or path/], + [ + { slot: 6, fact: 'product.rating.value', compare: 'number', tolerance: -1 }, + /tolerance must be a finite number >= 0/, + ], + [{ slot: 5, fact: 'breadcrumbs', compare: 'names', nameKey: '' }, /nameKey must be a non-empty string/], + [{ slot: 4, fact: 'product.offers', compare: 'skus', tuple: ['availability', 'price'] }, /tuple must name "sku"/], + [{ slot: 4, fact: 'product.offers', compare: 'skus', tuple: ['sku', 'sku', 'price'] }, /each at most once/], + [{ slot: 4, fact: 'product.offers', compare: 'skus', tuple: ['sku', 'color'] }, /tuple\[1\] must be/], + [{ slot: 4, fact: 'product.offers', compare: 'skus', tuple: ['sku'] }, /at least one of "availability" or "price"/], + ['title', /must be an object/], + ]; + for (const [entry, pattern] of cases) { + const warnings = []; + const rule = mapped({ fields: [entry, { slot: 0, fact: 'title', compare: 'text' }] }, warnings); + assert.ok(rule, `the rule survives ${JSON.stringify(entry)}`); + assert.deepEqual( + rule.pageCheck.fields.map((field) => field.label), + ['0:title'], + `only the bad entry drops: ${JSON.stringify(entry)}` + ); + assert.equal(warnings.length, 1, JSON.stringify(warnings)); + assert.match(warnings[0], pattern); + assert.match(warnings[0], /entry dropped/); + } + // A duplicate (slot, fact) pair drops the SECOND one. + const warnings = []; + const dup = mapped( + { + fields: [ + { slot: 0, fact: 'title', compare: 'text' }, + { slot: 0, fact: 'title', compare: 'text' }, + ], + }, + warnings + ); + assert.equal(dup.pageCheck.fields.length, 1); + assert.match(warnings[0], /maps slot 0 to title a second time/); + // An unknown key is warned about but does not drop the entry (it changes nothing it compares). + const typo = []; + const kept = mapped({ fields: [{ slot: 6, fact: 'product.rating.value', compare: 'number', tolerence: 1 }] }, typo); + assert.equal(kept.pageCheck.fields.length, 1); + assert.deepEqual(kept.pageCheck.fields[0].options, { tolerance: 0 }); + assert.match(typo[0], /unknown key\(s\) tolerence ignored/); + // A non-array is ignored whole, and with nothing else usable the block is too. + const notArray = []; + assert.equal(mapped({ fields: { slot: 0 } }, notArray).pageCheck, null); + assert.match(notArray[0], /pageCheck\.fields must be an array/); +}); + +test('pageCheck with fields: an INVALID claim pair drops the pair and keeps the fields; alone it drops the block', () => { + const warnings = []; + const rule = mapped( + { priceFrom: 3, availableFrom: 99, fields: [{ slot: 0, fact: 'title', compare: 'text' }] }, + warnings + ); + assert.equal(rule.pageCheck.priceFrom, null); + assert.equal(rule.pageCheck.fields.length, 1); + assert.match(warnings[0], /the price\/availability claim is ignored \(fields and ignoreChanges still apply\)/); + // Without fields or ignoreChanges the pair is required, exactly as before. + const alone = []; + assert.equal(mapped({}, alone).pageCheck, null); + assert.match(alone[0], /pageCheck ignored/); + // A block of every entry bad is no block at all. + assert.equal(mapped({ fields: [{ slot: 99, fact: 'title', compare: 'text' }] }).pageCheck, null); +}); + +test('pageCheck.ignoreChanges: entries validated alone, sorted, and a warning when it covers every slot', () => { + const warnings = []; + const rule = mapped({ ignoreChanges: [7, 2, 'x', 99, 2] }, warnings); + assert.deepEqual(rule.pageCheck.ignoreChanges, [2, 7]); + assert.equal(rule.pageCheck.fields.length, 0); + assert.equal(warnings.length, 2); + assert.match(warnings[0], /ignoreChanges\[2\] must be an integer index/); + assert.match(warnings[1], /ignoreChanges\[3\]/); + const everything = []; + mapped({ ignoreChanges: [0, 1, 2, 3, 4, 5, 6, 7] }, everything); + assert.match(everything[0], /lists EVERY extract slot/); + const notArray = []; + assert.equal(mapped({ ignoreChanges: 3 }, notArray).pageCheck, null); + assert.match(notArray[0], /ignoreChanges must be an array/); +}); + +test('mappings and ignore lists are NOT in the rule fingerprint — adding them re-baselines nothing', () => { + const fp = (pageCheck) => compileProbeRules([{ ...MAPPED_BASE, pageCheck }])[0].fingerprint; + const none = fp(undefined); + assert.equal(fp({ enabled: true, fields: [{ slot: 0, fact: 'title', compare: 'text' }] }), none); + assert.equal(fp({ enabled: true, ignoreChanges: [7] }), none); + assert.equal( + fp({ enabled: true, priceFrom: 3, availableFrom: 4, fields: [{ slot: 0, fact: 'h1', compare: 'text' }] }), + none + ); +}); + +// ---- comparators ---------------------------------------------------------------------------------- + +const field = (compare, fact, options = {}) => ({ slot: 0, fact, compare, options, label: `0:${fact}` }); +const PAGE_URL = 'https://www.example.com/product/prd-1/red-shoe.jsp'; +const verdict = (f, api, facts, vocabulary = null) => compareField(f, api, facts, { pageUrl: PAGE_URL, vocabulary }); + +test('text: exact after NFC + whitespace collapse + trim — no case folding, no HTML stripping', () => { + const title = field('text', 'title'); + assert.equal(verdict(title, 'Red Shoe | Example', { title: 'Red Shoe | Example' }), true); + assert.equal(verdict(title, ' Red Shoe \n| Example ', { title: 'Red Shoe | Example' }), true, 'whitespace'); + assert.equal(verdict(title, 'Café', { title: 'Café' }), true, 'NFC'); + assert.equal(verdict(title, 'red shoe | example', { title: 'Red Shoe | Example' }), false, 'case matters'); + // MEASURED QUIRK: the endpoint's description carries markup, and the page's meta tag repeats the + // RAW string. Stripping the markup would turn a 100% match into a disagreement on every page. + const meta = field('text', 'metaDescription'); + const raw = 'Soft leather.
  • Care
    • '; + assert.equal(verdict(meta, raw, { metaDescription: raw }), true); + assert.equal( + verdict(meta, raw, { metaDescription: 'Soft leather.Care' }), + false, + 'the stripped text is a different string' + ); + // No claim on either side is NOT a disagreement. + assert.equal(verdict(title, null, { title: 'x' }), null); + assert.equal(verdict(title, ' ', { title: 'x' }), null); + assert.equal(verdict(title, 42, { title: '42' }), null, 'a non-string endpoint value is not guessed at'); + assert.equal(verdict(title, 'x', { title: null }), null); + assert.equal(verdict(title, 'x', null), null); + // product.* reads through the product object. + assert.equal(verdict(field('text', 'product.brand'), 'Acme', { product: { brand: 'Acme' } }), true); + assert.equal(verdict(field('text', 'product.name'), 'Acme', { product: null }), null); +}); + +test('path: URL path only — size parameters, origin and fragment ignored; relative values resolve against the page', () => { + const image = field('path', 'product.image'); + // MEASURED QUIRK: the endpoint's image URL differs from the page's on EVERY page, by size params alone. + assert.equal( + verdict(image, 'https://media.example.com/is/image/shoe_1?w=350&h=350', { + product: { image: 'https://media.example.com/is/image/shoe_1?w=1000&h=1000' }, + }), + true + ); + assert.equal( + verdict(image, '//cdn.example.com/is/image/shoe_1?w=350', { + product: { image: 'https://media.example.com/is/image/shoe_1' }, + }), + true, + 'protocol-relative, and a different host: the path is what is compared' + ); + assert.equal( + verdict(image, 'https://media.example.com/is/image/shoe_2?w=350', { + product: { image: 'https://media.example.com/is/image/shoe_1?w=1000' }, + }), + false + ); + const canonical = field('path', 'canonical'); + // An SEO URL from the endpoint is usually a PATH; the canonical is absolute. + assert.equal(verdict(canonical, '/product/prd-1/red-shoe.jsp', { canonical: `${PAGE_URL}#top` }), true); + assert.equal(verdict(canonical, '/product/prd-1/blue-shoe.jsp', { canonical: PAGE_URL }), false); + assert.equal( + verdict(canonical, '/p/caf%C3%A9~x', { canonical: 'https://www.example.com/p/café%7Ex' }), + true, + 'encoding' + ); + assert.equal( + verdict(canonical, '/a/', { canonical: 'https://www.example.com/a' }), + false, + 'a trailing slash is a different path' + ); + assert.equal(verdict(canonical, '', { canonical: PAGE_URL }), null); + assert.equal(verdict(canonical, '/x', { canonical: null }), null); +}); + +test('number: numeric equality, "4.0" meets 4, with an optional tolerance; no number is no claim', () => { + const value = field('number', 'product.rating.value', { tolerance: 0 }); + const count = field('number', 'product.rating.count', { tolerance: 0 }); + // MEASURED QUIRK: the endpoint states the rating as a string, the page as a number. + assert.equal(verdict(value, '4.0', { product: { rating: [4, 120] } }), true); + assert.equal(verdict(value, '4.3', { product: { rating: [4, 120] } }), false); + assert.equal(verdict(count, 120, { product: { rating: [4, 120] } }), true); + assert.equal(verdict(count, '121', { product: { rating: [4, 120] } }), false); + assert.equal( + verdict(field('number', 'product.rating.value', { tolerance: 0.05 }), 4.25, { product: { rating: [4.3, 1] } }), + true + ); + // A null count stays null — never Number(null) === 0. + assert.equal(verdict(count, 0, { product: { rating: [4, null] } }), null); + assert.equal(verdict(count, null, { product: { rating: [4, 0] } }), null); + assert.equal(verdict(count, '1,234', { product: { rating: [4, 1234] } }), null, 'unparseable is no claim'); + assert.equal(verdict(value, '', { product: { rating: [0, 1] } }), null); +}); + +const offers = (...list) => ({ product: { offers: list } }); + +test('priceSet: a single endpoint price must be printed; for a list, every printed price must still be offered', () => { + const set = field('priceSet', 'product.offers'); + assert.equal(verdict(set, 35.99, offers(['1', '35.99', 'USD', 'InStock'])), true, 'number vs string'); + // A single endpoint price (a "lowest price") need only be AMONG the page's variant prices. + assert.equal( + verdict(set, 19.99, offers(['1', '19.99', 'USD', 'InStock'], ['2', '24.99', 'USD', 'InStock'])), + true, + 'scalar is contained' + ); + // MEASURED QUIRK: a page's structured data can stop at N offers while the endpoint lists every + // variant, so a price that exists only on an unlisted variant is absent from the page by + // construction. That must agree — equality here re-rendered the product on every pass, forever. + assert.equal( + verdict(set, [10, 12, 15], offers(['1', '10', 'USD', 'InStock'], ['2', '12', 'USD', 'InStock'])), + true, + 'truncated page lists a subset of the endpoint set' + ); + // A price the page prints that the endpoint no longer offers IS a disagreement. + assert.equal( + verdict(set, [10, 15], offers(['1', '10', 'USD', 'InStock'], ['2', '12', 'USD', 'InStock'])), + false, + 'page prints a price the endpoint dropped' + ); + assert.equal(verdict(set, '35.9', offers(['1', '35.90', 'USD', 'InStock'])), true); + assert.equal( + verdict( + set, + [19.99, 24.99, 19.99], + offers(['1', '19.99', 'USD', 'InStock'], ['2', '24.99', 'USD', 'InStock'], ['3', '19.99', 'USD', 'OutOfStock']) + ), + true, + 'distinct sets' + ); + assert.equal( + verdict(set, [19.99], offers(['1', '19.99', 'USD', 'InStock'], ['2', '24.99', 'USD', 'InStock'])), + false + ); + assert.equal(verdict(set, 29.99, offers(['1', '35.99', 'USD', 'InStock'])), false); + // MEASURED QUIRK: the endpoint's lowest price is null exactly when the product is out of stock. + // That is no claim, never "disagrees with every price on the page". + assert.equal(verdict(set, null, offers(['1', '35.99', 'USD', 'OutOfStock'])), null); + // A hole on either side means the sets cannot be compared. + assert.equal(verdict(set, [19.99, null], offers(['1', '19.99', 'USD', 'InStock'])), null); + assert.equal(verdict(set, 19.99, offers(['1', '19.99', 'USD', 'InStock'], ['2', null, 'USD', 'InStock'])), null); + assert.equal(verdict(set, [], offers(['1', '19.99', 'USD', 'InStock'])), null); + assert.equal(verdict(set, 19.99, offers()), null); + assert.equal(verdict(set, 19.99, { product: { offers: null } }), null); + assert.equal(verdict(set, 'USD 19.99', offers(['1', '19.99', 'USD', 'InStock'])), null); +}); + +test('names: the endpoint list against the page list TAIL — a leading home crumb needs no config', () => { + const crumbs = field('names', 'breadcrumbs', { nameKey: 'name' }); + const page = { breadcrumbs: ['Home', 'Shoes', 'Running Shoes'] }; + // MEASURED QUIRK: the page opens with a site-home crumb the endpoint omits. + assert.equal(verdict(crumbs, ['Shoes', 'Running Shoes'], page), true); + assert.equal( + verdict(crumbs, [{ name: 'Shoes', url: '/c/s' }, { name: 'Running Shoes' }], page), + true, + 'objects via nameKey' + ); + assert.equal(verdict(field('names', 'breadcrumbs', { nameKey: 'label' }), [{ label: 'Running Shoes' }], page), true); + assert.equal(verdict(crumbs, ['Home', 'Shoes', 'Running Shoes'], page), true, 'the whole list is its own tail'); + assert.equal(verdict(crumbs, ['Boots', 'Running Shoes'], page), false); + assert.equal(verdict(crumbs, ['Running Shoes', 'Shoes'], page), false, 'order matters'); + assert.equal(verdict(crumbs, ['X', 'Home', 'Shoes', 'Running Shoes'], page), false, 'the page is missing a level'); + assert.equal(verdict(crumbs, [], page), null); + assert.equal(verdict(crumbs, [{ url: '/c/s' }], page), null, 'a crumb with no name is no claim'); + assert.equal(verdict(crumbs, 'Shoes', page), null); + assert.equal(verdict(crumbs, ['Shoes'], { breadcrumbs: null }), null); +}); + +test('skus: per-SKU availability verdict and price, over the INTERSECTION of SKUs only', () => { + const skus = field('skus', 'product.offers', { tuple: { sku: 0, availability: 1, price: 2 } }); + const page = offers(['111', '19.99', 'USD', 'InStock'], ['222', '24.99', 'USD', 'OutOfStock']); + assert.equal( + verdict( + skus, + [ + ['111', 'In Stock', 19.99], + ['222', 'Out of Stock', null], + ], + page + ), + true + ); + // MEASURED QUIRK: the page lists at most 50 SKU offers where the endpoint lists more — a SKU only + // the endpoint has is not a disagreement. + assert.equal( + verdict( + skus, + [ + ['111', 'In Stock', 19.99], + ['999', 'In Stock', 5], + ], + page + ), + true + ); + assert.equal(verdict(skus, [['111', 'Out of Stock', 19.99]], page), false, 'availability verdicts differ'); + assert.equal(verdict(skus, [['111', 'In Stock', 17.99]], page), false, 'both state a price, and they differ'); + // Out of stock with a null endpoint price: availability still compares, price is no claim. + assert.equal(verdict(skus, [['222', 'Out of Stock', null]], page), true); + // An unrecognized word is no verdict — nothing compared, no claim. + assert.equal(verdict(skus, [['222', 'Backordered', null]], page), null); + // The rule's vocabulary applies on the endpoint side. + const vocabulary = { available: new Set(['backordered']), unavailable: new Set() }; + assert.equal(verdict(skus, [['222', 'Backordered', null]], page, vocabulary), false); + assert.equal(verdict(skus, [['999', 'In Stock', 1]], page), null, 'no SKU in common'); + // A numeric SKU meets its string form; a duplicated SKU is ambiguous and skipped. + assert.equal(verdict(skus, [[111, 'In Stock', 19.99]], page), true); + assert.equal( + verdict( + skus, + [ + ['111', 'In Stock', 1], + ['111', 'In Stock', 19.99], + ], + page + ), + null + ); + // A custom tuple order. + const priceFirst = field('skus', 'product.offers', { tuple: { sku: 1, availability: null, price: 0 } }); + assert.equal(verdict(priceFirst, [[19.99, '111', 'ignored']], page), true); + assert.equal(verdict(priceFirst, [[18.99, '111']], page), false); + // Garbage entries are skipped, not guessed at. + assert.equal(verdict(skus, [null, 'x', ['111', 'In Stock', 19.99]], page), true); + assert.equal(verdict(skus, 'x', page), null); +}); + +test('fieldCaughtUp: agreement with the new value — and for skus, every CHANGED SKU must itself be proven', () => { + const title = field('text', 'title'); + assert.equal(fieldCaughtUp(title, 'Old', 'New', { title: 'New' }, {}), true); + assert.equal(fieldCaughtUp(title, 'Old', 'New', { title: 'Old' }, {}), false); + assert.equal(fieldCaughtUp(title, 'Old', 'New', null, {}), false, 'no record is no evidence'); + + const skus = field('skus', 'product.offers', { tuple: { sku: 0, availability: 1, price: 2 } }); + const page = offers(['111', '19.99', 'USD', 'InStock'], ['222', '24.99', 'USD', 'InStock']); + const before = [ + ['111', 'In Stock', 21.99], + ['222', 'In Stock', 24.99], + ['999', 'In Stock', 5], + ]; + // 111 repriced and the page shows the new price: caught up. + assert.equal( + fieldCaughtUp( + skus, + before, + [ + ['111', 'In Stock', 19.99], + ['222', 'In Stock', 24.99], + ['999', 'In Stock', 5], + ], + page + ), + true + ); + // 999 repriced — a SKU the page does not list. The SKUs the page DOES list all agree, so the + // intersection verdict is true; reading that as "caught up" would swallow the change. + const offPage = [ + ['111', 'In Stock', 21.99], + ['222', 'In Stock', 24.99], + ['999', 'In Stock', 4], + ]; + assert.equal(compareField(skus, offPage, page, {}), false, '(111 still disagrees here)'); + const offPageOnly = [ + ['111', 'In Stock', 19.99], + ['222', 'In Stock', 24.99], + ['999', 'In Stock', 4], + ]; + const beforeOffPage = [ + ['111', 'In Stock', 19.99], + ['222', 'In Stock', 24.99], + ['999', 'In Stock', 5], + ]; + assert.equal(compareField(skus, offPageOnly, page, {}), true); + assert.equal( + fieldCaughtUp(skus, beforeOffPage, offPageOnly, page, {}), + false, + 'an unprovable changed SKU is not caught up' + ); + // A SKU gone from the endpoint while the page still lists it: not caught up. And one the page + // never listed: the page agrees with the old list as well as the new one, so nothing proves it was + // re-rendered — that triggers, as every removal did before fields. + const removed = [['111', 'In Stock', 19.99]]; + assert.equal( + fieldCaughtUp( + skus, + [ + ['111', 'In Stock', 19.99], + ['222', 'In Stock', 24.99], + ], + removed, + page + ), + false + ); + assert.equal( + fieldCaughtUp( + skus, + [ + ['111', 'In Stock', 19.99], + ['333', 'In Stock', 1], + ], + removed, + page + ), + false + ); + // A changed tuple that cannot be keyed is never provable. + assert.equal( + fieldCaughtUp( + skus, + [['111', 'In Stock', 19.99]], + [ + ['111', 'In Stock', 19.99], + [null, 'In Stock', 1], + ], + page + ), + false + ); +}); + +test('fieldCaughtUp: a change the comparator cannot SEE is never caught up — the page agrees with both values', () => { + // A crumb's URL changed inside the endpoint's objects; only names are compared, and the page's links + // may be exactly what is stale. + const crumbs = field('names', 'breadcrumbs', { nameKey: 'name' }); + const trail = { breadcrumbs: ['Home', 'Shoes'] }; + assert.equal( + fieldCaughtUp(crumbs, [{ name: 'Shoes', url: '/c/1' }], [{ name: 'Shoes', url: '/c/2' }], trail, {}), + false + ); + assert.equal(fieldCaughtUp(crumbs, [{ name: 'Boots' }], [{ name: 'Shoes' }], trail, {}), true, 'a visible change is'); + // A cache-busting query parameter under `path`, a whitespace edit under `text`. + const image = field('path', 'product.image'); + const shown = { product: { image: 'https://media.example.com/i/a.jpg?w=1000' } }; + assert.equal( + fieldCaughtUp(image, 'https://media.example.com/i/a.jpg?v=2', 'https://media.example.com/i/a.jpg?v=3', shown, {}), + false + ); + assert.equal(fieldCaughtUp(field('text', 'title'), 'Red Shoe', 'Red Shoe', { title: 'Red Shoe' }, {}), false); + // A SKU whose tuple changed only in a position the mapping skips. + const skus = field('skus', 'product.offers', { tuple: { sku: 0, availability: null, price: 1 } }); + const offersShown = offers(['111', '19.99', 'USD', 'InStock']); + assert.equal(fieldCaughtUp(skus, [['111', 19.99, 7]], [['111', 19.99, 6]], offersShown, {}), false); + assert.equal(fieldCaughtUp(skus, [['111', 21.99, 7]], [['111', 19.99, 6]], offersShown, {}), true); + // Two SKUs change at once: one the page lists (and shows the new price for), one it does not. The + // visible change makes the page disagree with the OLD list, so only the per-SKU proof stops the + // off-page change being swallowed with it. + const pair = offers(['111', '19.99', 'USD', 'InStock']); + assert.equal( + fieldCaughtUp( + skus, + [ + ['111', 21.99, 7], + ['999', 5, 1], + ], + [ + ['111', 19.99, 7], + ['999', 4, 1], + ], + pair, + {} + ), + false + ); + // ...and likewise a second SKU whose tuple moved only in a skipped position. + const both = offers(['111', '19.99', 'USD', 'InStock'], ['222', '9.99', 'USD', 'InStock']); + assert.equal( + fieldCaughtUp( + skus, + [ + ['111', 21.99, 7], + ['222', 9.99, 3], + ], + [ + ['111', 19.99, 7], + ['222', 9.99, 2], + ], + both, + {} + ), + false + ); + assert.equal( + fieldCaughtUp( + skus, + [ + ['111', 21.99, 7], + ['222', 9.99, 3], + ], + [ + ['111', 19.99, 7], + ['222', 9.99, 3], + ], + both, + {} + ), + true + ); + // A value the endpoint did not state before: the page showing the new one is evidence. + assert.equal(fieldCaughtUp(field('text', 'title'), null, 'Red Shoe', { title: 'Red Shoe' }, {}), true); +}); + +test('changedSlots: which slots moved, or null when a literal or a shape change makes that unknowable', () => { + assert.deepEqual(changedSlots('[1,"a",[1,2]]', '[1,"b",[1,3]]'), [1, 2]); + assert.deepEqual(changedSlots('[1,{"x":1}]', '[2,{"x":1}]'), [0]); + assert.equal(changedSlots('unavailable', '[1,2]'), null, 'a status-signal literal is a state, not slots'); + assert.equal(changedSlots('[1,2]', 'unavailable'), null); + assert.equal(changedSlots('[1,2]', '[1,2,3]'), null, 'different shapes'); + assert.equal(changedSlots('[1,2]', '[1,2]'), null, 'nothing moved: not a change of any slot'); +}); + +test('compareField never throws on a corrupted record — it is no claim', () => { + const cases = [ + [field('text', 'product.name'), 'x', { product: 'str' }], + [field('number', 'product.rating.value', { tolerance: 0 }), 1, { product: { rating: 'x' } }], + [field('priceSet', 'product.offers'), 1, { product: { offers: 'x' } }], + [ + field('skus', 'product.offers', { tuple: { sku: 0, availability: 1, price: 2 } }), + [['1', 'x', 1]], + { product: { offers: [5] } }, + ], + [field('names', 'breadcrumbs', { nameKey: 'name' }), ['a'], { breadcrumbs: [1, 2] }], + ]; + for (const [f, api, facts] of cases) + assert.equal(compareField(f, api, facts, { pageUrl: PAGE_URL }), null, f.compare); +}); + +// ---- the stored page record ---------------------------------------------------------------------- + +test('canonicalPageFacts: the contract shape in a FIXED key order — extra keys and wrong types dropped', () => { + const a = canonicalPageFacts({ + h1: 'Red Shoe', + title: 'Red Shoe | Example', + extra: 'dropped', + product: { offers: [['1', '9.99', 'USD', 'InStock'], 'junk'], rating: [4.5, '12'], name: 'Red Shoe', brand: 7 }, + breadcrumbs: ['Home', 'Shoes'], + canonical: PAGE_URL, + metaDescription: '', + }); + assert.deepEqual(Object.keys(a), ['canonical', 'title', 'metaDescription', 'h1', 'product', 'breadcrumbs']); + assert.deepEqual(a, { + canonical: PAGE_URL, + title: 'Red Shoe | Example', + metaDescription: null, // empty string: one way to say "no claim" + h1: 'Red Shoe', + product: { + name: 'Red Shoe', + brand: null, // wrong type + image: null, + rating: [4.5, null], // a string count is not the contract's number + offers: [ + ['1', '9.99', 'USD', 'InStock'], + [null, null, null, null], + ], + }, + breadcrumbs: ['Home', 'Shoes'], + }); + // Equal facts serialize to equal bytes whatever order the renderer sent them in. + const b = canonicalPageFacts({ + canonical: PAGE_URL, + breadcrumbs: ['Home', 'Shoes'], + h1: 'Red Shoe', + title: 'Red Shoe | Example', + product: { name: 'Red Shoe', rating: [4.5, '12'], offers: [['1', '9.99', 'USD', 'InStock'], 'junk'] }, + }); + assert.equal(JSON.stringify(b), JSON.stringify(a)); + // Nothing claimable is null, and a trail with a non-string crumb is no trail. + assert.equal(canonicalPageFacts({ title: '', product: { name: null }, breadcrumbs: [] }), null); + assert.equal(canonicalPageFacts(null), null); + assert.equal(canonicalPageFacts('facts'), null); + assert.equal(canonicalPageFacts({ title: 't', breadcrumbs: ['a', 5] }).breadcrumbs, null); +}); + +test('serializePageFacts REFUSES a record over the bound rather than truncating it; parsePageFacts round-trips', () => { + const small = serializePageFacts({ title: 'Red Shoe' }); + assert.equal(small.refused, false); + assert.deepEqual(parsePageFacts(small.json), { + canonical: null, + title: 'Red Shoe', + metaDescription: null, + h1: null, + product: null, + breadcrumbs: null, + }); + // 200 offers — the renderer's own cap — fit. + const many = Array.from({ length: 200 }, (_, i) => [`sku-${i}`, '19.99', 'USD', 'InStock']); + assert.equal(serializePageFacts({ product: { offers: many } }).refused, false); + // A pathological page does not: stored as null, and the size reported. + const huge = serializePageFacts({ breadcrumbs: Array.from({ length: 30 }, () => 'x'.repeat(2000)) }); + assert.equal(huge.refused, true); + assert.equal(huge.json, null); + assert.ok(huge.bytes > PAGE_FACTS_MAX_BYTES); + assert.equal(PAGE_FACTS_MAX_BYTES, 16 * 1024); + assert.deepEqual(serializePageFacts(null), { json: null, bytes: 0, refused: false }); + for (const junk of [null, '', '{', '[1]', '"s"', 5]) assert.equal(parsePageFacts(junk), null, String(junk)); +}); diff --git a/packages/plugin/test/renderQueueVariants.test.js b/packages/plugin/test/renderQueueVariants.test.js index f7963a87..2dfd78f6 100644 --- a/packages/plugin/test/renderQueueVariants.test.js +++ b/packages/plugin/test/renderQueueVariants.test.js @@ -289,6 +289,67 @@ test('every device rendered: pages stored per device, ONE reschedule, ONE outcom assert.equal(leased(A), false, 'the lease is released'); }); +/** + * The page record rides with the claim: from the first variant that ran its extraction, and ONLY when + * the result replaced every default device's page — a partial render cannot vouch for the device page + * it did not replace. + */ +const withPageRecord = async (t) => { + const { applyOptions } = await import('../src/config.js'); + const probeRows = new Map(); + const original = globalThis.databases.probe_state.ProbeState; + globalThis.databases.probe_state.ProbeState = makeResourceBase(probeRows); + applyOptions({ + changeProbe: { + enabled: true, + rules: [ + { + label: 'pdp', + pathPattern: '^/product/', + source: 'request', + request: { urlTemplate: 'https://api.example.com/x', method: 'POST', body: '{}' }, + extract: ['title'], + pageCheck: { enabled: true, fields: [{ slot: 0, fact: 'title', compare: 'text' }] }, + }, + ], + }, + }); + t.after(() => { + globalThis.databases.probe_state.ProbeState = original; + applyOptions({}); + }); + return probeRows; +}; + +test('every device rendered: the page record is stored from the result, stamped with the pages’ own lastCached', async (t) => { + const probeRows = await withPageRecord(t); + seedUrlRow(); + await claim(); + await postVariants(A, [ + rendered('desktop', 'd', { pageFacts: { title: 'Red Shoe' } }), + rendered('mobile', 'm', { pageFacts: { title: 'Red Shoe' } }), + ]); + const record = probeRows.get(A); + assert.equal(JSON.parse(record.pageFacts).title, 'Red Shoe'); + assert.equal( + record.pageClaimAt.getTime(), + stores.prerenderedPage.get(key(A, 'desktop')).lastCached, + 'the record describes exactly the render stored beside it' + ); +}); + +test('a PARTIAL render (one device failed) stores no page facts — the other device page is older', async (t) => { + const probeRows = await withPageRecord(t); + seedUrlRow(); + probeRows.set(A, { url: A, pageFacts: '{"title":"from an older render"}' }); + await claim(); + await postVariants(A, [ + rendered('desktop', 'fresh', { pageFacts: { title: 'Red Shoe' } }), + { deviceType: 'mobile', outcome: 'error', reason: 'error', statusCode: 500, headers: {} }, + ]); + assert.equal(probeRows.get(A).pageFacts, null, 'retired, not kept and not replaced by one device’s view'); +}); + test('a device that failed puts the URL in the fast lane while the device that rendered is stored', async () => { seedUrlRow(); await claim();