diff --git a/package-lock.json b/package-lock.json index 9b93a599..4380f60b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8556,7 +8556,7 @@ }, "packages/console": { "name": "@harperfast/prerender-console", - "version": "0.12.0", + "version": "0.13.0", "license": "Apache-2.0", "dependencies": { "undici": "^7.18.2" @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.64.1", + "version": "0.65.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/console/package.json b/packages/console/package.json index 62f62687..5d2c05b3 100644 --- a/packages/console/package.json +++ b/packages/console/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-console", - "version": "0.12.0", + "version": "0.13.0", "type": "module", "description": "Standalone Harper component serving the prerender management console UI, proxying to a prerender deployment's /prerender_admin API", "license": "Apache-2.0", diff --git a/packages/console/src/admin/views/probe.js b/packages/console/src/admin/views/probe.js index bea46569..639acf3a 100644 --- a/packages/console/src/admin/views/probe.js +++ b/packages/console/src/admin/views/probe.js @@ -134,6 +134,7 @@ const OUTCOMES = [ ['fresh', 'Skipped (fresh)', '#6b7488'], ['probed', 'Probes', '#3d8cff'], ['seeded', 'Seeded', '#8a93a6'], + ['rebaselined', 'Re-baselined', '#7c8cc4'], ['changed', 'Changed', '#f0a02a'], ['page_mismatch', 'Page mismatch', '#22b8cf'], ['triggered', 'Triggered', '#10a87e'], @@ -413,6 +414,7 @@ function drift(ctx) { const changed = totalOf('changed'); const failed = totalOf('failed'); const seeded = totalOf('seeded'); + const rebaselined = totalOf('rebaselined'); const deferred = totalOf('deferred'); const triggered = totalOf('triggered'); const trips = totalOf('canary_trip'); @@ -432,7 +434,12 @@ function drift(ctx) { // Compared = probes that had a baseline to compare against. Seeds and failures had none, so // including them in the denominator understates the drift rate by exactly the seeding backlog. - const compared = Math.max(0, probed - seeded - failed); + // RE-BASELINED rows had one and it was not comparable — it was taken under a different rule — + // so the plugin stores the new observation and compares nothing. They belong on the same side + // as seeds. Leaving them in is not a rounding error: the pass right after a rule edit can be + // almost entirely re-baselined, and the card would read "0% changed" over a population that was + // never compared at all, which is indistinguishable from a completely static origin. + const compared = Math.max(0, probed - seeded - rebaselined - failed); // Through `ratioOf` like every other ÷ figure in this console, and NOT because these two can be // null — `sumValues` reduces from 0 over finite products, so they are always numbers. It is that // the guard belongs at the site by convention rather than by an argument a reader has to @@ -583,8 +590,11 @@ function drift(ctx) { '“Failed” the origin caused, “Page mismatch” overlays the outcome buckets (a mismatched row is ', 'also inside “Changed” or the unchanged remainder), and “Skipped” sits outside “Probes” entirely ', 'because those rows were never attempted. “Changed” is measured against the probes that HAD a ', - 'baseline; seeds and failures are excluded from that denominator because neither compared ', - 'anything. “Page mismatch” stays at zero unless a rule sets pageCheck AND the render fleet posts ', + 'baseline; seeds, re-baselined rows and failures are excluded from that denominator because none ', + 'of them compared anything. “Re-baselined” is a row whose stored baseline was taken under a ', + 'DIFFERENT rule — a rule edit, not a content change — so the plugin stored the new observation ', + 'and compared nothing; expect one pass of them after any rule edit, and read a steady count as a ', + 'rule that keeps changing. “Page mismatch” stays at zero unless a rule sets pageCheck AND the render fleet posts ', 'its pages’ offers (browser 1.20.0+) — enabled against an older fleet it records nothing, and the ', 'plugin log says so hourly.', ]), @@ -639,7 +649,10 @@ function sweepCard(ctx, status) { } if (hasCounters) { - const compared = Math.max(0, (last.probed ?? 0) - (last.seeded ?? 0) - (last.failed ?? 0)); + const compared = Math.max( + 0, + (last.probed ?? 0) - (last.seeded ?? 0) - (last.rebaselined ?? 0) - (last.failed ?? 0) + ); body.push( kv([ [ diff --git a/packages/console/test/probeView.test.js b/packages/console/test/probeView.test.js index c6da79a8..3e814ffb 100644 --- a/packages/console/test/probeView.test.js +++ b/packages/console/test/probeView.test.js @@ -196,6 +196,28 @@ test('the change rate is measured against what was COMPARED, not against every p assert.match(changed.textContent, /of 2\.0k compared/); }); +test('re-baselined rows leave the compared denominator, so a rule edit cannot read as a static origin', async () => { + // The pass right after a rule edit: the plugin re-baselines every row it could not compare — + // stored under a different rule fingerprint — so almost nothing was actually compared. Counting + // those as compared reports "0% changed" over a population nothing looked at, which is exactly + // how a completely static catalogue reads. + const analytics = { + ...ANALYTICS, + series: [ + passes('probed', 4, 1000), + passes('seeded', 4, 50), + passes('rebaselined', 4, 900), + passes('failed', 4, 10), + passes('changed', 4, 4), + ], + }; + const ctx = await ready({ analytics }); + const changed = tile(ctx, 'Changed'); + // 4,000 probed − 200 seeded − 3,600 re-baselined − 40 failed = 160 compared, and 16 changed. + assert.match(changed.textContent, /of 160 compared/); + assert.match(changed.textContent, /10%/); +}); + test('a pass counter is summed by VALUE — counting emits would report the number of passes', async () => { const ctx = await ready(); // 4 passes × 1000 = 4,000 probes. Summing `count` would say 4. diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 9dc10d2f..d4877612 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,17 +121,17 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert). | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: diff --git a/packages/plugin/package.json b/packages/plugin/package.json index b03926c0..1afb54f3 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.64.1", + "version": "0.65.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 69ed9744..04f6975f 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -898,8 +898,19 @@ export const configSchema = group('Prerender plugin configuration.', { ' request.body request body string (e.g. "{}").\n' + ' extract (request mode, required) value paths into the JSON response, e.g. ' + '"payload.products[0].prices[0].salePrice" — the extracted values ARE the watched content; ' + - 'everything else in the response is ignored. An extraction where every path yields null is a ' + + 'everything else in the response is ignored. A path may end at an object or array, which is ' + + 'signed whole, and `[*]` projects the rest of the path over every element of an array ' + + '("payload.products[0].variants[*].availability" -> one value per variant, in the endpoint’s ' + + 'order) — the way to watch per-variant state without signing fields that move on their own ' + + '(inventory counters, store data). An extraction where every path yields null is a ' + 'FAILED probe, never a new signature, so an endpoint shape change cannot mass-trigger.\n' + + ' EDITING A RULE. The rule’s observation (endpoint, method, headers, body, extract, ' + + 'statusSignals) is fingerprinted and stored beside every baseline. Change any of it and each ' + + 'matched URL is RE-BASELINED on its next probe — new observation stored, nothing compared, ' + + 'nothing triggered, not counted by the canary — instead of the new signature shape reading as ' + + '100% of the corpus changing at once. A rule edit therefore costs one pass without detection ' + + 'for that rule and needs no dry-run cycle. Label, pathPattern, invalidateScope and pageCheck ' + + 'are not part of the fingerprint: they change what is matched or done, not what is observed.\n' + ' statusSignals optional [{ status, signature, contains? }] — statuses this endpoint uses ' + 'to SAY something rather than to fail, mapped to a fixed signature. An endpoint that answers ' + 'a legitimate state with an error status (most usefully "no longer available" as a 4xx with ' + @@ -932,9 +943,16 @@ export const configSchema = group('Prerender plugin configuration.', { 'page\u2019s offers. A page yielding no Product offers records nothing, exactly as a failed ' + 'probe changes nothing, so a markup change cannot make every page look like a disagreement \u2014 ' + 'and each dimension compares only when BOTH sides make a readable claim (availability must ' + - 'reduce to a recognized schema.org verdict on the page and a boolean at the endpoint; price ' + - 'must parse as a number on the page), so an unrecognized vocabulary or price format degrades ' + - 'to detecting nothing rather than expiring everything. ' + + 'reduce to a recognized schema.org verdict on the page, and at the endpoint to a boolean, an ' + + 'availability word, or a `[*]` list of per-variant words; price must parse as a number on the ' + + 'page), so an unrecognized vocabulary or price format degrades to detecting nothing rather ' + + 'than expiring everything. Availability words are matched after dropping case and separators ' + + '(InStock, IN_STOCK, "In Stock" and https://schema.org/InStock are one word); the built-in ' + + 'vocabulary is Google’s (InStock, InStoreOnly, OnlineOnly, LimitedAvailability, Available / ' + + 'OutOfStock, SoldOut, Discontinued, Unavailable), and `pageCheck.availableValues` / ' + + '`pageCheck.unavailableValues` (arrays of words) extend or override it per rule for an endpoint ' + + 'with its own vocabulary. A per-variant list reads in stock when ANY variant is, exactly as the ' + + 'page’s offers are read. ' + 'Detection is one extra node-local write per render and no extra origin traffic.\n' + ' invalidateScope optional invalidation scope ("all" or "route::") the canary ' + 'records on a mass change. Empty = the canary detects and logs only.\n' + @@ -956,9 +974,36 @@ export const configSchema = group('Prerender plugin configuration.', { 'time lost to backoff are absorbed as they happen. `ratePerSecond` stays a hard ceiling. ' + 'A target that cannot be met at the ceiling is reported (`probe_cycle_behind`) rather ' + 'than silently missed — which is the whole point of the mode.\n\n' + - 'Switching is safe in both directions and takes effect on the next config apply; a pass ' + + '"anchored" runs ONE full pass a day, starting at `anchorTime` in `anchorTimezone` and ' + + 'paced to `anchorWindow` (0 = as fast as `ratePerSecond` allows). For an origin whose content ' + + 'moves on a schedule — a retailer whose prices change only at its own midnight — this puts ' + + 'the walk right after the change instead of spreading it over the day, so detection latency ' + + 'is the pass length rather than up to a cycle, and the corpus is current for the day by the ' + + 'time the pass ends. No pass runs at boot in this mode (baselines persist; a restart waits ' + + 'for the anchor). The canary keeps its own cadence, so an off-schedule mass change is still ' + + 'caught; only the full walk is anchored.\n\n' + + 'Switching is safe in every direction and takes effect on the next config apply; a pass ' + 'in flight finishes under the rules it started with.', - { enum: ['interval', 'continuous'] } + { enum: ['interval', 'continuous', 'anchored'] } + ), + anchorTime: option( + '00:15', + 'ANCHORED MODE ONLY: local time of day ("HH:MM") the daily pass starts, interpreted in ' + + '`anchorTimezone`. Put it just AFTER the origin’s scheduled change lands (a few minutes ' + + 'after its midnight, not at it) so the first probes see the new state rather than the ' + + 'tail of the old one.', + { nonEmpty: true } + ), + anchorTimezone: option('UTC', 'ANCHORED MODE ONLY: IANA timezone `anchorTime` is interpreted in.', { + nonEmpty: true, + }), + anchorWindow: option( + 0, + 'ANCHORED MODE ONLY: wall-clock budget the daily pass paces itself to, like `cycleTarget` ' + + 'for one pass. 0 (default) runs at the `ratePerSecond` ceiling and finishes as early as the ' + + 'agreed rate allows; set it to spread the pass deliberately (e.g. 6h) when the origin would ' + + 'rather see a lower steady rate than a short burst.', + { unit: 'ms', min: 0 } ), sweepInterval: option( DAY, diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index e110e8a1..49273a75 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -395,6 +395,14 @@ type ProbeState @table(database: "probe_state", replicate: false) @sealed { # was written would make the very page it certifies fail its own `lastCached >= basisAt` test. # Written in the SAME patch as `pageSignature`, so it costs no extra read and no extra write. pageClaimAt: Date + # The fingerprint of the RULE that produced `signature` (changeProbeSpec.js ruleFingerprint): + # endpoint, method, headers, body, extract paths, status signals. A baseline is only comparable + # to an observation made the same way, so a probe whose rule fingerprint differs from the stored + # one re-baselines the row instead of reading the new signature shape as a content change — + # which is what a rule edit used to look like, for 100% of matched URLs at once. Null on rows + # written before this column existed; those are compared as usual and stamped on their next + # quiet probe. Written by the probe only, in the same put/patch as `signature`. + ruleFingerprint: String } # Same anchor, same reason as CoordinationAnchor: ProbeState is `replicate: false`, and a diff --git a/packages/plugin/src/util/changeProbe.js b/packages/plugin/src/util/changeProbe.js index 2020c323..6a2cd27f 100644 --- a/packages/plugin/src/util/changeProbe.js +++ b/packages/plugin/src/util/changeProbe.js @@ -50,7 +50,7 @@ import { gunzipSync } from 'node:zlib'; import { config, onConfigApplied } from '../config.js'; import { metrics } from '../metrics.js'; import { fnv1a32 } from './hash.js'; -import { epochMsOf, currentMinuteMs, MINUTE, SECOND } from './time.js'; +import { epochMsOf, currentMinuteMs, getNextTimeOfDay, DAY, MINUTE, SECOND } from './time.js'; import { getResidencyByUrl } from './residency.js'; import { resolveEffectiveInterval } from './routeClass.js'; import { writeSchedules } from './renderSchedule.js'; @@ -135,8 +135,9 @@ const newStats = () => ({ examined: 0, // rows scanned owned: 0, // rows this node owns matched: 0, // owned rows a rule matched (suppressed excluded) - probed: 0, // probes attempted = seeded + unchanged + changed + failed + probed: 0, // probes attempted = seeded + rebaselined + unchanged + changed + failed seeded: 0, // first observation stored, nothing compared + rebaselined: 0, // baseline was taken under a DIFFERENT rule fingerprint: observation stored, nothing compared or triggered (a rule edit, not a content change) unchanged: 0, changed: 0, triggered: 0, // changes that scheduled a re-render @@ -324,7 +325,7 @@ export const triggerRevalidate = async (row) => { const readSignature = async (url) => { const row = await probeStateTable().get({ id: url, - select: ['url', 'signature', 'probedAt', 'pageSignature', 'pageClaimAt'], + select: ['url', 'signature', 'probedAt', 'pageSignature', 'pageClaimAt', 'ruleFingerprint'], }); if (!row) return null; // A Date column can surface as a Date, an epoch number, a string, or — the trap this coercion @@ -339,6 +340,8 @@ const readSignature = async (url) => { probedAt, pageSignature: row.pageSignature ?? null, pageClaimAt: row.pageClaimAt ?? null, + // null for a row written before fingerprints existed — read as "the rule in force", see processOne. + fingerprint: row.ruleFingerprint ?? null, }; }; // The probe's write must NEVER carry the page claim through a whole-row put: `pageSignature` @@ -351,8 +354,12 @@ const readSignature = async (url) => { // the seeding write is by definition to a URL with no row yet; a patch there would make every // first observation a silent no-op. A row the render path creates between the read and that put // loses at most one baseline to the race and re-seeds on the next pass. -export const writeSignature = (url, signature, { rowExists = false, clearClaim = false } = {}) => { +export const writeSignature = (url, signature, { rowExists = false, clearClaim = false, fingerprint = null } = {}) => { const fields = { signature, probedAt: new Date() }; + // The rule that made this observation (changeProbeSpec.js ruleFingerprint), so the next pass + // can tell a rule edit from a content change. Written with EVERY baseline, including the + // one-time stamp of a pre-fingerprint row; a signature is never left beside a stale fingerprint. + if (fingerprint !== null) fields.ruleFingerprint = fingerprint; // BOTH HALVES OF THE PAIR, always. `pageClaimAt` is the render `pageSignature` came from, and the // two are only meaningful together — a claim with no basis cannot be scoped to a device page, and // a basis with no claim describes nothing. Clearing one and leaving the other is inert today @@ -576,6 +583,27 @@ export const runProbePass = async ({ } distressStreak = 0; + // RULE CHANGED, NOT CONTENT. A baseline is only comparable to an observation made the same + // way (changeProbeSpec.js ruleFingerprint). A stored fingerprint that is not this rule's + // means the rule was edited since the baseline was taken: store the new observation, compare + // nothing, trigger nothing, and keep the row out of the canary's verdict — a config edit must + // never read as a mass change. Before this, every rule edit produced a differently-shaped + // signature for 100% of matched URLs at once, and the only safe way to make one was a full + // dry-run cycle. Rows from before fingerprints existed carry none and are compared as usual: + // the rule that wrote them is the rule in force (a deploy that changed the rule would have + // reseeded them the old way), and they are stamped on their next quiet probe below, so the + // NEXT rule edit costs nothing either. + const ruleChanged = + Boolean(stored?.signature) && + stored.fingerprint !== null && + stored.fingerprint !== undefined && + stored.fingerprint !== rule.fingerprint; + if (ruleChanged) { + stats.rebaselined++; + await write(row.url, observed, { rowExists: true, fingerprint: rule.fingerprint }); + return; + } + // ROUND-TRIP BLINDNESS. Everything below compares the origin to the origin, so a value // that changed and changed BACK between two passes is invisible — and a render that landed // inside that window left a page carrying the transient value. `pageSignature` is what the @@ -601,7 +629,7 @@ export const runProbePass = async ({ const signatureChanged = Boolean(stored?.signature) && stored.signature !== observed; // Bucket by SIGNATURE outcome alone, BEFORE the page check influences control flow: - // `probed = seeded + unchanged + changed + failed` is the documented invariant, and the + // `probed = seeded + rebaselined + unchanged + changed + failed` is the documented invariant, and the // canary's denominator is changed + unchanged — a page-mismatch row that skipped both // would silently shrink the mass-change sample right when claims are most likely to be // stale. `pageMismatch` OVERLAYS these buckets; it never replaces them. @@ -612,8 +640,16 @@ export const runProbePass = async ({ if (!stored?.signature) { // First observation: baseline it, trigger nothing — the page's content is not known // to have changed, the probe just hadn't seen it before. - await write(row.url, observed, { rowExists: stored !== null }); - } else if (verificationArmed && stored.pageSignature) { + await write(row.url, observed, { rowExists: stored !== null, fingerprint: rule.fingerprint }); + return; + } + // One-time stamp of a pre-fingerprint row (see the rule-changed block above): one patch + // per legacy row, after which a converged corpus is back to paying no write per probe. + // Not an alternative to the verification below — both are due, neither stands in for the other. + if (stored.fingerprint === null || stored.fingerprint === undefined) { + await write(row.url, observed, { rowExists: true, fingerprint: rule.fingerprint }); + } + if (verificationArmed && stored.pageSignature) { // PROOF, not absence of news. Both conditions are load-bearing and neither is // redundant: // @@ -644,7 +680,7 @@ export const runProbePass = async ({ // the opposite case and is deliberately NOT cleared: nothing was expired, so the // disagreement still stands — in dry-run `pageMismatch` reads as a standing gauge of // disagreeing pages per pass, where armed it is a detection rate. - await write(row.url, observed, { rowExists: stored !== null }); + await write(row.url, observed, { rowExists: stored !== null, fingerprint: rule.fingerprint }); return; } if (stats.triggered >= maxTriggers) { @@ -663,7 +699,7 @@ export const runProbePass = async ({ // trigger budget on a page already expired and already filed. The next render writes a // fresh claim; until then there is nothing to compare, which is the correct "I don't // know" state. Folded into this write so it costs no second round trip. - await write(row.url, observed, { rowExists: stored !== null, clearClaim: true }); + await write(row.url, observed, { rowExists: stored !== null, clearClaim: true, fingerprint: rule.fingerprint }); } catch (e) { stats.errors++; globalThis.logger?.error?.(e, `[prerender] change-probe trigger failed for ${row.url}`); @@ -806,6 +842,7 @@ const emitStats = (stats, kind) => { try { metrics.changeProbe(stats.probed, 'probed'); metrics.changeProbe(stats.seeded, 'seeded'); + metrics.changeProbe(stats.rebaselined, 'rebaselined'); metrics.changeProbe(stats.changed, 'changed'); metrics.changeProbe(stats.triggered, 'triggered'); metrics.changeProbe(stats.deferred, 'deferred'); @@ -922,6 +959,10 @@ let lastCanary = null; let measuredSliceSize = null; const isContinuous = () => config.changeProbe.mode === 'continuous'; +const isAnchored = () => config.changeProbe.mode === 'anchored'; +// The armed-sweep marker for anchored mode carries the anchor itself, so editing the time or the +// zone reads as a mode change to `syncProbeTimers` (which compares markers) and re-arms the timer. +const anchorKey = () => `anchored:${config.changeProbe.anchorTime}|${config.changeProbe.anchorTimezone}`; /** * Shared pass limits. @@ -945,8 +986,23 @@ const passLimits = (dryRunOverride, { paced = false } = {}) => ({ // Continuous pacing, or zeroes — and zeroes are what make interval mode bit-identical: with // no cycle target `cycleRatePerSecond` is never consulted and the window is the one // `ratePerSecond` has always implied. - cycleTarget: paced && isContinuous() ? config.changeProbe.cycleTarget : 0, - sliceSize: paced && isContinuous() ? (measuredSliceSize ?? 0) : 0, + // Continuous paces to its cycle; an anchored pass paces to its window (0 = the ceiling, the + // default: a daily pass exists to finish, not to spread); an interval pass is never paced. + cycleTarget: !paced + ? 0 + : isContinuous() + ? config.changeProbe.cycleTarget + : isAnchored() + ? config.changeProbe.anchorWindow + : 0, + // The denominator the cycle target is honoured against. Anchored mode needs it as much as + // continuous does: without it `cycleRatePerSecond` has nothing to divide and returns Infinity, + // so `anchorWindow` would be read, reported, and then silently ignored — the pass would burst at + // the ceiling however the window was set, and `probe_cycle_behind` (which also requires a slice) + // would stay quiet about it. `measuredSliceSize` is maintained after every completed pass + // regardless of mode, so the first anchored pass paces at the ceiling and measures, and every + // pass after it honours the window. + sliceSize: paced && (isContinuous() || isAnchored()) ? (measuredSliceSize ?? 0) : 0, // The local governor is opt-in and orthogonal to the mode, so it is read from config rather // than gated on `isContinuous()` — an operator who wants it in interval mode has been warned // by the option's own documentation and may have reasons. It applies to the canary too: a @@ -1326,6 +1382,7 @@ export const changeProbeStatus = async () => { // has measured it yet, which is precisely when the pass runs at the ceiling — worth // being able to see, because "at the ceiling" otherwise looks identical to "behind". cycleTarget: isContinuous() ? config.changeProbe.cycleTarget : null, + nextAnchoredRunAt: isAnchored() && nextAnchorAt ? new Date(nextAnchorAt).toISOString() : null, sliceSize: isContinuous() ? (scheduler?.sliceSize ?? null) : null, }, load: { @@ -1383,10 +1440,81 @@ const clearProbeTimers = () => { if (bootTimer) clearTimeout(bootTimer); if (sweepTimer) clearInterval(sweepTimer); if (canaryTimer) clearInterval(canaryTimer); - bootTimer = sweepTimer = canaryTimer = null; + if (anchorTimer) clearTimeout(anchorTimer); + bootTimer = sweepTimer = canaryTimer = anchorTimer = null; + nextAnchorAt = null; stopContinuousLoop(); }; +/** + * ANCHORED MODE's driver: one pass a day, starting at `anchorTime` in `anchorTimezone`, then + * re-armed for the next occurrence once the pass has finished. + * + * This is the mode for an origin whose content moves on a schedule — a retailer whose prices + * change only at its own midnight, say. Continuous mode would spread the same probes evenly + * over 24h and so notice a midnight change anywhere from minutes to a day later; anchored mode + * starts the pass right after the change and (at the default window of 0, the rate ceiling) + * finishes it as fast as the agreed ceiling allows, so the corpus is current for the day by the + * time the pass ends. The canary keeps its own cadence in this mode, so a change that lands OFF + * the schedule is still caught as a mass change; only the full walk is anchored. + * + * The re-arm is computed AFTER the pass, against the clock: a pass that outran the day (longer + * than 24h at the ceiling) simply lands on the next anchor and skips none by accident, and a + * DST shift is absorbed by `getNextTimeOfDay` recomputing the offset at the target instant. + * `getNextTimeOfDay` is never more than a day out, so the delay fits setTimeout's signed 32-bit + * range without the clamp `sweepInterval` needs. + */ +let anchorTimer = null; +let nextAnchorAt = null; +const armAnchorTimer = () => { + if (anchorTimer) clearTimeout(anchorTimer); + let at; + try { + at = getNextTimeOfDay(config.changeProbe.anchorTime, config.changeProbe.anchorTimezone); + } catch (e) { + at = NaN; + logger.warn( + `[prerender] change-probe: anchorTimezone "${config.changeProbe.anchorTimezone}" is not usable (${e?.message ?? String(e)})` + ); + } + // An unusable anchor arms NOTHING. `setTimeout(fn, NaN)` fires at once, which would turn a typo + // in the timezone into a full-rate pass on every config apply; a warning and a null + // `nextAnchoredRunAt` on the admin surface is the failure mode that gets noticed and fixed. + if (!Number.isFinite(at)) { + nextAnchorAt = null; + logger.warn( + `[prerender] change-probe: anchored mode has no next run — fix anchorTime/anchorTimezone (the canary keeps running)` + ); + return; + } + // A NEXT RUN MUST BE IN THE FUTURE. On the spring-forward day the anchor's wall-clock time can + // be one that never occurs — 02:30 where 02:00 jumps to 03:00 — and `getNextTimeOfDay` resolves + // it to an instant that has already passed. Left alone that fires the pass an hour early and, + // worse, the re-arm at the end of the pass computes the same past instant again: the whole + // corpus is walked back to back at the ceiling rate until the hour is over. Pushing a stale + // anchor on by a day lands on the next real occurrence, because the offset is applied to the + // resolved instant rather than to the wall clock. + if (at <= Date.now()) at += DAY; + nextAnchorAt = at; + anchorTimer = setTimeout( + async () => { + anchorTimer = null; + nextAnchorAt = null; + try { + await runProbeSweepOnce(); + } catch (e) { + logger.error(e); + } + // Config is re-read here rather than captured: this is the boundary a live change acts on. + // A mode or anchor edit mid-pass has already re-armed through syncProbeTimers (the marker + // changed), so re-arming here as well would run two drivers. + if (config.changeProbe.enabled && isAnchored() && armedSweep === anchorKey()) armAnchorTimer(); + }, + Math.max(0, at - Date.now()) + ); + anchorTimer.unref?.(); +}; + /** * CONTINUOUS MODE's driver: finish a cycle, start the next one, forever. * @@ -1452,6 +1580,7 @@ const armIntervals = () => { // identically either way. An early return here would have silently disabled the mass-change // detector for anyone who turned continuous mode on. if (isContinuous()) void runContinuousLoop(); + else if (isAnchored()) armAnchorTimer(); else { sweepTimer = setInterval(() => runProbeSweepOnce().catch((e) => logger.error(e)), armedSweep); sweepTimer.unref?.(); @@ -1469,7 +1598,13 @@ const syncProbeTimers = () => { // the mode does, or `syncProbeTimers` sees no difference and leaves an interval timer running // after a switch to continuous (and vice versa). Tagging the mode into the key is what makes // the mode itself live. - const desiredSweep = !enabled ? null : isContinuous() ? 'continuous' : config.changeProbe.sweepInterval; + const desiredSweep = !enabled + ? null + : isContinuous() + ? 'continuous' + : isAnchored() + ? anchorKey() + : config.changeProbe.sweepInterval; const desiredCanary = enabled && config.changeProbe.canary.interval > 0 ? config.changeProbe.canary.interval : null; if (desiredSweep === armedSweep && desiredCanary === armedCanary) return; @@ -1497,7 +1632,10 @@ const syncProbeTimers = () => { void publishScheduler(); if (desiredSweep === null) return; - if (wasEnabled) { + // Anchored mode never runs a boot sweep: the whole point is that the pass starts at the anchor, + // and a restart at 15:00 must not walk the corpus at 15:05. Baselines persist across restarts + // (ProbeState), so nothing is lost by waiting for the anchor; the canary is armed right away. + if (wasEnabled || isAnchored()) { armIntervals(); return; } diff --git a/packages/plugin/src/util/changeProbeSpec.js b/packages/plugin/src/util/changeProbeSpec.js index 77a97704..124bbd05 100644 --- a/packages/plugin/src/util/changeProbeSpec.js +++ b/packages/plugin/src/util/changeProbeSpec.js @@ -9,7 +9,8 @@ * * DELIBERATELY DEPENDENCY-FREE. config.js reports rule problems from `collectConfigWarnings`, * which is a pure function of config — it can import this module only because this module imports - * nothing of the runtime (no config, no tables, no globals). Keep it that way. + * nothing of the runtime (no config, no tables, no globals). Keep it that way. (`./hash.js` is a + * pure function too; that is the one import.) * * WHY A SIGNATURE AND NOT A DIFF. The probe never needs to know WHAT changed, only WHETHER the * watched fields changed — so the observation is reduced to a canonical string @@ -24,6 +25,8 @@ * re-renders of pages that did not change. */ +import { fnv1a32 } from './hash.js'; + const VALID_SOURCES = new Set(['request', 'document']); // No HEAD: extraction parses the response body, and a HEAD probe has none — it would validate // here and then fail on every single probe, which is the config shape this compiler exists to refuse. @@ -182,7 +185,33 @@ const compileRule = (raw, index, warn) => { `extract (0-${extract.length - 1}) — pageCheck ignored` ); } else { - pageCheck = { priceFrom: pc.priceFrom, availableFrom: pc.availableFrom }; + pageCheck = { priceFrom: pc.priceFrom, availableFrom: pc.availableFrom, vocabulary: null }; + // The rule's own availability words, on top of the built-in schema.org/retail sets. + // Tokenized the same way the endpoint's values will be, so "In Stock" and "IN_STOCK" in + // config meet "in stock" in a response. Invalid lists drop the vocabulary, not pageCheck: + // the built-in words still apply. + const words = (key) => { + const list = pc[key]; + if (list === undefined || list === null) return []; + if (!Array.isArray(list) || list.some((w) => typeof w !== 'string' || !availabilityToken(w))) { + warn(`change-probe ${label}: pageCheck.${key} must be an array of availability words — ignored`); + return null; + } + return list.map(availabilityToken); + }; + const available = words('availableValues'); + const unavailable = words('unavailableValues'); + if (available && unavailable && (available.length || unavailable.length)) { + const overlap = available.filter((w) => unavailable.includes(w)); + if (overlap.length) { + warn( + `change-probe ${label}: pageCheck.availableValues and .unavailableValues both list ` + + `${overlap.join(', ')} — vocabulary ignored` + ); + } else { + pageCheck.vocabulary = { available: new Set(available), unavailable: new Set(unavailable) }; + } + } } } } else if (pc.enabled) { @@ -194,7 +223,7 @@ const compileRule = (raw, index, warn) => { } } - return { + const rule = { label, pathPattern, patternSource: raw.pathPattern, @@ -205,6 +234,49 @@ const compileRule = (raw, index, warn) => { statusSignals, pageCheck, }; + rule.fingerprint = ruleFingerprint(rule); + return rule; +}; + +/** + * What a rule OBSERVES, hashed: the endpoint, how it is asked, which values are taken from the + * answer, and which statuses stand in for values. Stored beside every baseline (ProbeState. + * ruleFingerprint) so the sweep can tell "the origin changed" from "the rule changed". + * + * WHY THIS EXISTS. A stored signature is only comparable to an observation made THE SAME WAY. + * Edit a rule's extract list, move it to another endpoint, add a header that changes which + * backend answers — and the next probe of every matched URL produces a differently-shaped + * signature. Without this, that read as 100% of the corpus changing in one pass: every URL a + * spurious re-render (bounded by maxTriggersPerSweep, so most of them deferred and retried + * forever), and the canary a certain trip, invalidating the rule's whole scope over a config + * edit. The only safe way to change a rule was a full dry-run cycle first. With the fingerprint, + * a mismatch re-baselines that URL — observation stored, nothing compared, nothing triggered, + * and it does not count toward the canary's verdict — so a rule edit costs one pass of blindness + * for the edited rule and nothing else. + * + * WHAT IS IN IT, AND WHAT IS NOT. Everything that shapes the observed value: source, URL + * template, method, headers (a cookie or header can route to a different backend), body, + * extract paths, status signals (a signal is a value). NOT the label, the pathPattern (which + * URLs match, not what is seen of them), invalidateScope or pageCheck (how an observation is + * acted on, not what it is). Header keys are sorted so a reordered YAML map is not a new rule. + */ +export const ruleFingerprint = (rule) => { + const headers = Object.fromEntries( + Object.entries(rule.request?.headers ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ); + const observed = + rule.source === 'request' + ? { + source: 'request', + url: rule.request.urlTemplate, + method: rule.request.method, + headers, + body: rule.request.body ?? null, + extract: rule.extract, + signals: rule.statusSignals, + } + : { source: 'document', headers, signals: rule.statusSignals }; + return fnv1a32(JSON.stringify(observed)).toString(16).padStart(8, '0'); }; /** @@ -231,11 +303,35 @@ const canonicalPrice = (value) => { * endpoint on EVERY pass and hard-expires every matched page, forever — the one failure mode this * feature must not have — so an unrecognized vocabulary degrades to "detects nothing" instead. */ -const AVAILABLE = new Set(['instock', 'instoreonly', 'onlineonly', 'limitedavailability']); -const UNAVAILABLE = new Set(['outofstock', 'soldout', 'discontinued']); -const availabilityVerdict = (raw) => { +const AVAILABLE = new Set(['instock', 'instoreonly', 'onlineonly', 'limitedavailability', 'available']); +const UNAVAILABLE = new Set(['outofstock', 'soldout', 'discontinued', 'unavailable']); + +/** + * A vocabulary token: the last path segment of a schema.org URL form, lower-cased, with every + * separator dropped — so `https://schema.org/InStock`, `InStock`, `IN_STOCK`, `in-stock` and the + * plain retail phrase `In Stock` all reduce to `instock`. Endpoints spell availability every one + * of these ways; the plugin is not the place to know which retailer uses which. + */ +export const availabilityToken = (raw) => + String(raw) + .split('/') + .filter(Boolean) + .pop() + ?.toLowerCase() + .replace(/[^a-z0-9]/g, '') ?? ''; + +/** + * The verdict for one availability word: true, false, or null for no verdict. `vocabulary` is a + * rule's own `{ available, unavailable }` token sets (pageCheck.availableValues / + * unavailableValues), consulted BEFORE the built-in schema.org/retail sets so an operator can + * both extend the vocabulary and override a built-in reading for their endpoint. + */ +const availabilityVerdict = (raw, vocabulary = null) => { if (typeof raw !== 'string') return null; - const token = raw.split('/').filter(Boolean).pop()?.toLowerCase() ?? ''; + const token = availabilityToken(raw); + if (!token) return null; + if (vocabulary?.available.has(token)) return true; + if (vocabulary?.unavailable.has(token)) return false; if (AVAILABLE.has(token)) return true; if (UNAVAILABLE.has(token)) return false; return null; @@ -276,22 +372,54 @@ export const pageClaimFromOffers = (flat) => { return JSON.stringify([[...prices].sort(), inStock]); }; +/** + * The endpoint's availability field as a verdict. Three shapes are readable, everything else is + * NO claim (null) — a field this plugin cannot read must degrade to "detects nothing", never to + * a guess that disagrees with every page on every pass: + * + * boolean (or 'true'/'false') the verdict itself + * string an availability word, read by `availabilityVerdict` — the + * schema.org forms, the plain retail phrases ("In Stock", + * "Out of Stock", "Sold Out") and the rule's own vocabulary + * array (a `[*]` projection) per-variant words: in stock when ANY variant is, out of stock + * only when every readable variant is and none is unreadable — + * the same reduction `pageClaimFromOffers` applies to the page's + * offers, so the two sides answer the same question + */ +const availabilityClaim = (raw, vocabulary) => { + if (raw === true || raw === 'true') return true; + if (raw === false || raw === 'false') return false; + if (typeof raw === 'string') return availabilityVerdict(raw, vocabulary); + if (Array.isArray(raw)) { + let sawAvailable = false; + let sawUnavailable = false; + let sawUnrecognized = false; + for (const item of raw) { + const verdict = availabilityClaim(item, vocabulary); + if (verdict === true) sawAvailable = true; + else if (verdict === false) sawUnavailable = true; + // EVERY element that yields no verdict counts as unreadable, including a null one. A + // `[*]` projection writes exactly null where the path could not be walked — a variant + // that simply has no availability field yet — so skipping nulls let a list of one + // sold-out variant and one unreadable variant answer a confident "out of stock" that + // the endpoint never said. That claim is compared against the page's own, and a + // disagreement expires a page: the cost of guessing here is paid in origin traffic on + // products that were never out of stock. + else sawUnrecognized = true; + } + return sawAvailable ? true : sawUnavailable && !sawUnrecognized ? false : null; + } + return null; +}; + /** * The same claim shape, projected from the values the probe just extracted from the endpoint. - * Availability is a claim only for an unambiguous boolean (true/'true'/false/'false') — any other - * type is a mapping the operator got wrong or a field this plugin cannot read, and guessing turns - * every probe of every page into a disagreement. Null when neither field yields a claim. + * Null when neither field yields a claim. */ export const apiClaimOf = (values, pageCheck) => { if (!pageCheck || !Array.isArray(values)) return null; const price = canonicalPrice(values[pageCheck.priceFrom]); - const availableRaw = values[pageCheck.availableFrom]; - const available = - availableRaw === true || availableRaw === 'true' - ? true - : availableRaw === false || availableRaw === 'false' - ? false - : null; + const available = availabilityClaim(values[pageCheck.availableFrom], pageCheck.vocabulary ?? null); if (price === null && available === null) return null; return JSON.stringify([price === null ? [] : [price], available]); }; @@ -413,13 +541,54 @@ export const substituteTemplate = (template, match) => /** * The value at a dot/bracket path (`payload.products[0].prices[0].salePrice`) or undefined. - * Tokens are plain property names and `[N]` numeric indexes; anything unreachable is undefined - * rather than a throw, because a probe response missing a branch is data, not a bug. + * Tokens are plain property names, `[N]` numeric indexes and `[*]` projections; anything + * unreachable is undefined rather than a throw, because a probe response missing a branch is + * data, not a bug. + * + * `[*]` projects the REST of the path over every element of an array (`SKUS[*].availability` -> + * `["In Stock", "Out of Stock", ...]`), positionally: element k of the result is element k of the + * array, with an unreachable branch as null so a variant that lost a field does not shift the + * others. Order is the endpoint's — a reorder reads as a change, exactly like a reordered array + * value in any other extracted field. This is how a rule watches per-variant state without + * signing the whole variant object, whose other fields (inventory counters, store data) move + * without the page moving. */ -export const valueAtPath = (value, path) => { +export const valueAtPath = (value, path) => walkPath(value, String(path).match(PATH_TOKEN) ?? [], 0); + +// `[*]` and `[N]` are tried BEFORE the bare-name alternative: `*` and digits are legal name +// characters to that alternative, so ordering it first would tokenize `[*]` as the name `*`. +const PATH_TOKEN = /\[\*\]|\[\d+\]|[^.[\]]+/g; + +// A total, stable order over projected elements of any shape — they may be strings, numbers, nulls +// or whole objects. Comparing their JSON keeps the sort deterministic across passes, which is the +// only property that matters here: the same multiset must always produce the same signature. +const byProjectedValue = (a, b) => { + const left = JSON.stringify(a) ?? 'null'; + const right = JSON.stringify(b) ?? 'null'; + return left < right ? -1 : left > right ? 1 : 0; +}; + +const walkPath = (value, tokens, from) => { let current = value; - for (const token of String(path).match(/[^.[\]]+|\[\d+\]/g) ?? []) { + for (let i = from; i < tokens.length; i++) { if (current === null || current === undefined) return undefined; + const token = tokens[i]; + if (token === '[*]') { + if (!Array.isArray(current)) return undefined; + const projected = current.map((element) => { + const value = walkPath(element, tokens, i + 1); + return value === undefined ? null : value; + }); + // ORDER IS NOT A CHANGE, so it is sorted out of the projection before it can become one. + // A signature is compared byte for byte, and plenty of endpoints return their variant + // array in whatever order the query came back in. Left positional, one such endpoint + // reads as 100% changed on EVERY pass — which on the canary cohort is not a wave of + // per-URL re-renders but a trip, and a bulk invalidation of the rule's whole scope. + // Sorting trades away the one case where a multiset is stable but its order is not: two + // variants swapping prices with each other is invisible. The offers on the page are the + // same set either way, which is what the signature is asking about. + return projected.sort(byProjectedValue); + } const key = token.startsWith('[') ? Number(token.slice(1, -1)) : token; current = current[key]; } diff --git a/packages/plugin/test/changeProbe.test.js b/packages/plugin/test/changeProbe.test.js index c77fea90..de866c69 100644 --- a/packages/plugin/test/changeProbe.test.js +++ b/packages/plugin/test/changeProbe.test.js @@ -109,17 +109,22 @@ const runPass = async ({ rows, answers, stored = {}, + // Per-URL stored rule fingerprint: absent = the current rule's (the steady state), null = a row + // from before fingerprints existed, any other string = a baseline taken under another rule. + fingerprints = {}, dryRun = false, maxTriggers = 100, owners = {}, ...overrides }) => { const { compileProbeRules } = await import('../src/util/changeProbeSpec.js'); + const rules = compileProbeRules(RULES_RAW); const written = []; + const writeOptions = []; const triggered = []; const stats = await changeProbe.runProbePass({ rows: stream(rows), - rules: compileProbeRules(RULES_RAW), + rules, ownerOf: (url) => owners[url] ?? 'node-a', hostname: 'node-a', probe: async (rule, url) => { @@ -128,8 +133,18 @@ const runPass = async ({ return answer ?? null; }, // The port returns the whole baseline now — `probedAt` is what the freshness skip reads. - read: async (url) => (stored[url] === undefined ? null : { signature: stored[url], probedAt: NaN }), - write: async (url, signature) => written.push({ url, signature }), + read: async (url) => + stored[url] === undefined + ? null + : { + signature: stored[url], + probedAt: NaN, + fingerprint: url in fingerprints ? fingerprints[url] : rules[0].fingerprint, + }, + write: async (url, signature, options) => { + written.push({ url, signature }); + writeOptions.push({ url, ...options }); + }, trigger: async (target) => triggered.push(target.url), dryRun, maxTriggers, @@ -138,7 +153,7 @@ const runPass = async ({ pause: async () => {}, ...overrides, }); - return { stats, written, triggered }; + return { stats, written, writeOptions, triggered, rules }; }; const URL_A = 'https://example.com/product/prd-a/'; @@ -182,6 +197,51 @@ test('the state machine: seed, unchanged, changed', async () => { assert.deepEqual(written.map((w) => w.url).sort(), [URL_A, URL_C].sort()); }); +test('a baseline taken under ANOTHER rule re-baselines: stored, not compared, not triggered', async () => { + const { stats, written, writeOptions, triggered, rules } = await runPass({ + rows: [row(URL_A), row(URL_B)], + stored: { [URL_A]: '[1]', [URL_B]: '[1]' }, + fingerprints: { [URL_A]: 'deadbeef' }, // B carries the current rule's fingerprint + answers: { [URL_A]: '[2]', [URL_B]: '[2]' }, + }); + // A: a rule edit produced a differently-shaped signature — that is not a content change. + // B: the same new value under the same rule IS a change. + assert.equal(stats.rebaselined, 1); + assert.equal(stats.changed, 1); + assert.equal(stats.triggered, 1); + assert.deepEqual(triggered, [URL_B]); + assert.deepEqual(written.map((w) => w.url).sort(), [URL_A, URL_B].sort()); + // Every baseline write carries the rule that made it, so the NEXT edit is recognised too. + assert.ok(writeOptions.every((w) => w.fingerprint === rules[0].fingerprint)); + assert.equal(stats.probed, stats.seeded + stats.rebaselined + stats.unchanged + stats.changed + stats.failed); +}); + +test('a pre-fingerprint row is compared as usual and stamped once when quiet', async () => { + const { stats, written, writeOptions, triggered, rules } = await runPass({ + rows: [row(URL_A), row(URL_B), row(URL_C)], + stored: { [URL_A]: '[1]', [URL_B]: '[1]', [URL_C]: '[1]' }, + fingerprints: { [URL_A]: null, [URL_B]: null }, // legacy rows; C is already stamped + answers: { [URL_A]: '[1]', [URL_B]: '[2]', [URL_C]: '[1]' }, + }); + // A: unchanged, legacy -> one stamp write. B: changed, legacy -> compared like any row (a + // legacy baseline was taken by the rule in force). C: unchanged and stamped -> no write at all, + // the converged-corpus guarantee. + assert.equal(stats.unchanged, 2); + assert.equal(stats.changed, 1); + assert.equal(stats.rebaselined, 0); + assert.deepEqual(triggered, [URL_B]); + assert.deepEqual(written.map((w) => w.url).sort(), [URL_A, URL_B].sort()); + assert.ok(writeOptions.every((w) => w.fingerprint === rules[0].fingerprint)); +}); + +test('the canary verdict ignores re-baselined rows, like seeds', async () => { + const verdict = changeProbe.canaryVerdict( + { changed: 0, unchanged: 0, rebaselined: 500, seeded: 0 }, + { threshold: 0.1, minSample: 10 } + ); + assert.deepEqual(verdict, { tripped: false, compared: 0, fraction: null }); +}); + test('dry run counts and re-baselines but never triggers', async () => { const { stats, written, triggered } = await runPass({ rows: [row(URL_A)], @@ -1130,6 +1190,89 @@ test('scheduler: mode is live — switching re-arms rather than leaving the old await applyProbeConfig({ enabled: false }); }); +test('scheduler: anchored mode arms a daily timer keyed on the anchor, runs no boot sweep, and re-arms on edit', async (t) => { + t.mock.timers.enable({ apis: ['setInterval', 'setTimeout'] }); + + await applyProbeConfig({ + enabled: true, + mode: 'anchored', + anchorTime: '03:00', + anchorTimezone: 'UTC', + startDelay: 0, + startJitter: 1, + }); + changeProbe.startChangeProbeScheduler(); + assert.equal(changeProbe.probeTimerState().armedSweep, 'anchored:03:00|UTC'); + const status = await changeProbe.changeProbeStatus(); + assert.ok(status.sweep.nextAnchoredRunAt, 'the next run is published for the admin surface'); + const next = new Date(status.sweep.nextAnchoredRunAt); + assert.equal(next.getUTCHours(), 3); + assert.equal(next.getUTCMinutes(), 0); + assert.ok(next.getTime() > Date.now() && next.getTime() - Date.now() <= 24 * 60 * 60 * 1000); + // No boot sweep: with startDelay 0 an interval/continuous boot would have started a pass by now. + assert.equal(status.sweep.running, false); + + // Editing the anchor is a mode change to the scheduler: the marker moves, the timer re-arms. + await applyProbeConfig({ + enabled: true, + mode: 'anchored', + anchorTime: '04:30', + anchorTimezone: 'UTC', + startDelay: 0, + startJitter: 1, + }); + assert.equal(changeProbe.probeTimerState().armedSweep, 'anchored:04:30|UTC'); + + // Back to interval mode (not continuous, which would start a pass that outlives the test). + await applyProbeConfig({ enabled: true, sweepInterval: 60_000, startDelay: 0, startJitter: 1 }); + assert.equal(changeProbe.probeTimerState().armedSweep, 60_000); + + t.mock.timers.reset(); +}); + +test('scheduler: an anchor inside the spring-forward hour still arms a FUTURE run', async (t) => { + // 2026-03-08, America/New_York: 02:00 EST jumps to 03:00 EDT, so an anchor of 02:30 names a + // wall-clock time that does not occur that day and resolves to an instant already past. Armed + // as-is, the pass fires early AND every re-arm at the end of a pass computes the same past + // instant — the whole corpus walked back to back at the ceiling rate until the hour is over. + // The host's own zone is pinned to UTC for the duration: the resolver reads it, so without this + // the instant it returns — and therefore whether the bug reproduces at all — depends on wherever + // the test happens to run. + const hostTz = process.env.TZ; + process.env.TZ = 'UTC'; + t.mock.timers.enable({ apis: ['setInterval', 'setTimeout', 'Date'] }); + try { + // Three instants inside the skipped hour. Every one of them resolved into the past before + // the guard, and each would have re-armed to the same past instant on every completion. + for (const now of ['2026-03-08T06:35:00Z', '2026-03-08T06:45:00Z', '2026-03-08T06:55:00Z']) { + t.mock.timers.setTime(Date.parse(now)); + await applyProbeConfig({ + enabled: true, + mode: 'anchored', + anchorTime: '02:30', + anchorTimezone: 'America/New_York', + startDelay: 0, + startJitter: 1, + }); + changeProbe.startChangeProbeScheduler(); + + const status = await changeProbe.changeProbeStatus(); + const next = new Date(status.sweep.nextAnchoredRunAt).getTime(); + assert.ok( + next > Date.now(), + `at ${now} the next anchored run must be in the future, got ${new Date(next).toISOString()}` + ); + assert.equal(status.sweep.running, false, `at ${now} it must not fire on the spot`); + await applyProbeConfig({ enabled: true, sweepInterval: 60_000, startDelay: 0, startJitter: 1 }); + } + } finally { + t.mock.timers.reset(); + if (hostTz === undefined) delete process.env.TZ; + else process.env.TZ = hostTz; + } + await applyProbeConfig({ enabled: false }); +}); + test('scheduler: the canary stays armed in continuous mode', async (t) => { t.mock.timers.enable({ apis: ['setInterval', 'setTimeout'] }); diff --git a/packages/plugin/test/changeProbeSpec.test.js b/packages/plugin/test/changeProbeSpec.test.js index b52a271d..326e5e16 100644 --- a/packages/plugin/test/changeProbeSpec.test.js +++ b/packages/plugin/test/changeProbeSpec.test.js @@ -27,6 +27,8 @@ import { pageClaimFromOffers, apiClaimOf, claimsDisagree, + ruleFingerprint, + availabilityToken, } from '../src/util/changeProbeSpec.js'; const REQUEST_RULE = { @@ -296,7 +298,7 @@ test('pageCheck compiles only with in-bounds indices, and only for source "reque extract: ['a', 'b', 'c', 'd'], }; const ok = compileProbeRules([{ ...base, pageCheck: { enabled: true, priceFrom: 2, availableFrom: 3 } }]); - assert.deepEqual(ok[0].pageCheck, { priceFrom: 2, availableFrom: 3 }); + assert.deepEqual(ok[0].pageCheck, { priceFrom: 2, availableFrom: 3, vocabulary: null }); // out of bounds -> dropped whole, rule survives (a half-applied mapping compares the wrong column) const oob = compileProbeRules([{ ...base, pageCheck: { enabled: true, priceFrom: 2, availableFrom: 9 } }]); @@ -420,12 +422,193 @@ test('apiClaimOf projects through the mapping; absent mapped fields yield no cla assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'false'], pc)), [['35.99'], false]); assert.equal(apiClaimOf([null, null, null, null], pc), null); assert.equal(apiClaimOf([1, 2, 3, true], null), null); - // A non-boolean availability field (a count, a status string) is a mapping the operator got - // wrong or a shape the plugin cannot read — availability becomes NO claim rather than a guess - // that would disagree with every page on every pass. Price alone still projects. - assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'IN_STOCK'], pc)), [['35.99'], null]); + // An availability WORD is a claim: endpoints spell it every way (schema.org form, constant + // case, the plain retail phrase) and all of them reduce to one token. + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'IN_STOCK'], pc)), [['35.99'], true]); + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'In Stock'], pc)), [['35.99'], true]); + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'Out of Stock'], pc)), [['35.99'], false]); + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'Sold Out'], pc)), [['35.99'], false]); + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'https://schema.org/InStock'], pc)), [['35.99'], true]); + // A word outside the vocabulary, or a non-word (a count), is a shape the plugin cannot read — + // availability becomes NO claim rather than a guess that would disagree with every page on + // every pass. Price alone still projects. + assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 'PreOrder'], pc)), [['35.99'], null]); assert.deepEqual(JSON.parse(apiClaimOf([39.99, 35.99, 35.99, 7], pc)), [['35.99'], null]); - assert.equal(apiClaimOf([null, null, null, 'IN_STOCK'], pc), null); + assert.equal(apiClaimOf([null, null, null, 'Coming Soon'], pc), null); + // A `[*]` projection: in stock when ANY variant is, out only when every readable one is and + // none is unreadable — the reduction the page's offers get, so both sides answer one question. + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, ['Out of Stock', 'In Stock']], pc)), [['3.00'], true]); + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, ['Out of Stock', 'Out of Stock']], pc)), [['3.00'], false]); + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, ['Out of Stock', 'PreOrder']], pc)), [['3.00'], null]); + // A null element is an UNREADABLE variant, not an absent one: `[*]` writes null where the path + // could not be walked, so a sold-out variant beside one whose availability has not populated + // must not answer a confident "out of stock" the endpoint never gave. + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, ['Out of Stock', null]], pc)), [['3.00'], null]); + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, ['In Stock', null]], pc)), [['3.00'], true]); + assert.deepEqual(JSON.parse(apiClaimOf([1, 2, 3, []], pc)), [['3.00'], null]); +}); + +test('apiClaimOf consults the rule vocabulary before the built-in words', () => { + const pc = { + priceFrom: 0, + availableFrom: 1, + vocabulary: { available: new Set(['ships']), unavailable: new Set(['nope', 'instock']) }, + }; + assert.deepEqual(JSON.parse(apiClaimOf([9, 'Ships!'], pc)), [['9.00'], true]); + assert.deepEqual(JSON.parse(apiClaimOf([9, 'nope'], pc)), [['9.00'], false]); + // The rule's word beats the built-in reading — an endpoint may use a schema.org-looking word + // with its own meaning, and only the operator knows. + assert.deepEqual(JSON.parse(apiClaimOf([9, 'InStock'], pc)), [['9.00'], false]); + // Built-in words still apply for anything the rule did not name. + assert.deepEqual(JSON.parse(apiClaimOf([9, 'Sold Out'], pc)), [['9.00'], false]); + assert.equal(availabilityToken('https://schema.org/In-Stock'), 'instock'); + assert.equal(availabilityToken(' OUT_OF_STOCK '), 'outofstock'); + assert.equal(availabilityToken('///'), ''); +}); + +test('pageCheck.availableValues / unavailableValues compile into the vocabulary, or drop it alone', () => { + const base = { + pathPattern: '^/p/', + source: 'request', + request: { urlTemplate: 'https://x/$1' }, + extract: ['a', 'b'], + }; + const ok = compileProbeRules([ + { + ...base, + pageCheck: { + enabled: true, + priceFrom: 0, + availableFrom: 1, + availableValues: ['Ships Today', 'IN_STOCK'], + unavailableValues: ['Gone'], + }, + }, + ]); + assert.deepEqual([...ok[0].pageCheck.vocabulary.available].sort(), ['instock', 'shipstoday']); + assert.deepEqual([...ok[0].pageCheck.vocabulary.unavailable], ['gone']); + // Neither list -> no vocabulary object, built-in words only. + assert.equal( + compileProbeRules([{ ...base, pageCheck: { enabled: true, priceFrom: 0, availableFrom: 1 } }])[0].pageCheck + .vocabulary, + null + ); + // A word on both sides is a contradiction: the vocabulary drops, pageCheck stays. + const warnings = []; + const clash = compileProbeRules( + [ + { + ...base, + pageCheck: { enabled: true, priceFrom: 0, availableFrom: 1, availableValues: ['x'], unavailableValues: ['X'] }, + }, + ], + warnings + ); + assert.deepEqual(clash[0].pageCheck, { priceFrom: 0, availableFrom: 1, vocabulary: null }); + assert.match(warnings[0], /both list x/); + // A malformed list drops the vocabulary, not the page check. + const bad = compileProbeRules( + [{ ...base, pageCheck: { enabled: true, priceFrom: 0, availableFrom: 1, availableValues: 'yes' } }], + warnings + ); + assert.equal(bad[0].pageCheck.vocabulary, null); + assert.match(warnings[1], /availableValues must be an array/); +}); + +test('valueAtPath [*] projects the rest of the path over an array, positionally', () => { + const doc = { + variants: [{ a: { s: 'x' } }, { a: {} }, { a: { s: 'z' } }], + flat: [1, 2], + nested: [{ k: [{ v: 1 }, { v: 2 }] }, { k: [{ v: 3 }] }], + }; + // Projections come back SORTED — order is not a change, and an endpoint that returns its + // variants in an unstable order must not read as changed on every pass. + assert.deepEqual(valueAtPath(doc, 'variants[*].a.s'), ['x', 'z', null]); + assert.deepEqual(valueAtPath(doc, 'variants[*].a'), [{ s: 'x' }, { s: 'z' }, {}]); + assert.deepEqual(valueAtPath(doc, 'flat[*]'), [1, 2]); + assert.deepEqual(valueAtPath(doc, 'nested[*].k[*].v'), [[1, 2], [3]]); + // Past the end of a branch every element reads null, positionally, never a throw. + assert.deepEqual(valueAtPath(doc, 'variants[*].a.s.t'), [null, null, null]); + // Not an array -> unreachable, like any missing branch; a signature of all-null then fails the probe. + assert.equal(valueAtPath(doc, 'nope[*].a'), undefined); + assert.equal(valueAtPath({ variants: 'str' }, 'variants[*].a'), undefined); + assert.deepEqual(extractValues(doc, ['variants[*].a.s', 'missing[*]']), [['x', 'z', null], null]); + // `[*]` and `[N]` tokenize as brackets, never as the names `*` / digits. + assert.deepEqual(valueAtPath({ '*': 1, 'variants': [{ '*': 2 }] }, 'variants[*].*'), [2]); +}); + +test('a reordered array projects to the same value, so a reorder is not a change', () => { + // The failure this prevents: an endpoint with no stable variant order reads as 100% changed on + // every pass, which on the canary cohort is a trip and a bulk invalidation of the rule's scope. + const one = { + v: [ + { sku: 'a', av: 'In Stock' }, + { sku: 'b', av: 'Out of Stock' }, + ], + }; + const two = { + v: [ + { sku: 'b', av: 'Out of Stock' }, + { sku: 'a', av: 'In Stock' }, + ], + }; + assert.deepEqual(valueAtPath(one, 'v[*].av'), valueAtPath(two, 'v[*].av')); + assert.deepEqual(valueAtPath(one, 'v[*]'), valueAtPath(two, 'v[*]')); + // And the signature built from them agrees, which is the property that actually matters. + assert.equal( + JSON.stringify(extractValues(one, ['v[*].sku', 'v[*].av'])), + JSON.stringify(extractValues(two, ['v[*].sku', 'v[*].av'])) + ); + // A genuine change still reads as one. + const changed = { + v: [ + { sku: 'a', av: 'Out of Stock' }, + { sku: 'b', av: 'Out of Stock' }, + ], + }; + assert.notEqual(JSON.stringify(extractValues(one, ['v[*].av'])), JSON.stringify(extractValues(changed, ['v[*].av']))); +}); + +test('ruleFingerprint changes with what is observed and with nothing else', () => { + const base = { + label: 'a', + pathPattern: '^/p/(\\d+)', + source: 'request', + request: { + urlTemplate: 'https://x/$1', + method: 'GET', + headers: { Accept: 'application/json', Referer: 'https://x/' }, + }, + extract: ['a', 'b'], + statusSignals: [{ status: 400, contains: 'GONE', signature: 'gone' }], + invalidateScope: 'all', + pageCheck: { enabled: true, priceFrom: 0, availableFrom: 1 }, + }; + const fp = (raw) => compileProbeRules([raw])[0].fingerprint; + assert.match(fp(base), /^[0-9a-f]{8}$/); + // Same observation, different bookkeeping: label, pattern, scope, pageCheck, header ORDER. + assert.equal(fp({ ...base, label: 'b' }), fp(base)); + assert.equal(fp({ ...base, pathPattern: '^/q/(\\d+)' }), fp(base)); + assert.equal(fp({ ...base, invalidateScope: null }), fp(base)); + assert.equal(fp({ ...base, pageCheck: undefined }), fp(base)); + assert.equal( + fp({ ...base, request: { ...base.request, headers: { Referer: 'https://x/', Accept: 'application/json' } } }), + fp(base) + ); + // Different observation: endpoint, method, a header, the body, a path, a signal. + assert.notEqual(fp({ ...base, request: { ...base.request, urlTemplate: 'https://y/$1' } }), fp(base)); + assert.notEqual(fp({ ...base, request: { ...base.request, method: 'POST', body: '{}' } }), fp(base)); + assert.notEqual( + fp({ ...base, request: { ...base.request, headers: { ...base.request.headers, Cookie: 'c=1' } } }), + fp(base) + ); + assert.notEqual(fp({ ...base, extract: ['a', 'b', 'c'] }), fp(base)); + assert.notEqual(fp({ ...base, statusSignals: [] }), fp(base)); + // Document mode has its own fingerprint, distinct from any request rule. + assert.notEqual(fp({ pathPattern: '^/p/' }), fp(base)); + assert.equal(fp({ pathPattern: '^/p/' }), fp({ pathPattern: '^/other/', label: 'z' })); + // Direct call agrees with the compiled value. + assert.equal(ruleFingerprint(compileProbeRules([base])[0]), fp(base)); }); test('claimsDisagree survives a corrupted stored claim instead of ending the sweep', async () => {