From 06f75b2fdbc94062cac940a02f1ca9bd93aaa0d5 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 24 Sep 2026 19:07:23 -0400 Subject: [PATCH] =?UTF-8?q?perf(browser):=20one=20in-page=20evaluate=20for?= =?UTF-8?q?=20structuredOffers=20and=20pageFacts=20=E2=80=94=20one=20CDP?= =?UTF-8?q?=20round=20trip=20and=20one=20JSON-LD=20parse=20per=20render,?= =?UTF-8?q?=20outputs=20unchanged;=20v1.38.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The renderer read the page's two claims with two separate page.evaluate calls, each querying and JSON-parsing every JSON-LD block itself. extractPageClaims reads both in one evaluate: the blocks are queried once and each parsed at most once, on the first reader's demand and never past the furthest block either reader needs. The two readers' bodies are the 1.37.0 extractors verbatim apart from the block loop, so every output is unchanged, including structuredOffers' 64-char field slicing (the plugin's change probe stores those values). Each reader runs in its own in-page try/catch, so a throw in one still nulls only that claim; a failure of the evaluate itself posts null for both, never a failed render and never an absent field. test/pageClaimsParity.test.ts keeps both 1.37.0 extractors verbatim and holds the merged one to deep-equal and byte-identical output over every fixture of both suites, edge cases where the two walks diverge, hostile pages that break one reader, and a 400-document seeded fuzz corpus. Three deliberate mutations of the merged extractor each fail it. Co-Authored-By: Claude Opus 5.5 (1M context) --- package-lock.json | 2 +- packages/browser/README.md | 4 +- packages/browser/package.json | 2 +- packages/browser/src/pageFacts.ts | 523 ++++--- packages/browser/src/renderer.ts | 105 +- .../browser/test/pageClaimsParity.test.ts | 1245 +++++++++++++++++ packages/browser/test/pageFacts.test.ts | 44 +- 7 files changed, 1629 insertions(+), 296 deletions(-) create mode 100644 packages/browser/test/pageClaimsParity.test.ts diff --git a/package-lock.json b/package-lock.json index 7bb0c89d..07759f98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8516,7 +8516,7 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.37.0", + "version": "1.38.0", "license": "Apache-2.0", "dependencies": { "mqtt": "^5.10.4", diff --git a/packages/browser/README.md b/packages/browser/README.md index 35d4d0f0..5cce888e 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -142,7 +142,9 @@ carries facts the renderer read off the settled DOM (before `postProcess`), so t the cached page's own claims against its sources of truth without parsing HTML on its write path. Both fields share one wire contract: **absent** means the renderer predates the field (a consumer may alarm on it); **`null`** means the extraction ran and found nothing to claim or failed benignly — it is posted as -`null`, never omitted, and an extraction failure never fails the render. The extraction runs only on +`null`, never omitted, and an extraction failure never fails the render. Both are read in one in-page +pass (one `page.evaluate`, each JSON-LD block parsed once): a failure in one reader costs only that +field, and a failure of the pass itself posts `null` for both. The extraction runs only on the path that produces content, so a variant without content (a redirect, a verdict, an error) may carry neither — absence is meaningful only on a variant that carries content. diff --git a/packages/browser/package.json b/packages/browser/package.json index 0e169d35..116d24e8 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-browser", - "version": "1.37.0", + "version": "1.38.0", "type": "module", "description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.", "keywords": [ diff --git a/packages/browser/src/pageFacts.ts b/packages/browser/src/pageFacts.ts index 6ad358cb..95b8bea9 100644 --- a/packages/browser/src/pageFacts.ts +++ b/packages/browser/src/pageFacts.ts @@ -1,6 +1,7 @@ /** - * What the rendered page itself CLAIMS — its canonical, title, meta description, first heading, - * product JSON-LD and breadcrumb trail — read off the settled DOM and posted with each variant. + * What the rendered page itself CLAIMS — its schema.org offers (`structuredOffers`) and its SEO facts + * (canonical, title, meta description, first heading, product JSON-LD and breadcrumb trail) — read off + * the settled DOM and posted with each variant. * * WHY THE RENDERER. The consumer compares these against its own sources of truth to decide whether a * cached page is wrong (and only then re-render it). It deliberately never parses HTML on its hot @@ -10,7 +11,9 @@ * BOUNDS ARE REFUSALS, NEVER TRUNCATIONS. A truncated value would disagree with the consumer's source * of truth on every comparison, and a systematic disagreement means the consumer expires and * re-renders that page forever. So a value past its bound is reported as no claim at all (`null`), - * which degrades to exactly what "the page does not say" does: nothing to compare. + * which degrades to exactly what "the page does not say" does: nothing to compare. (`structuredOffers` + * predates this rule for its fields: it refuses past its offer cap but slices an over-long field to 64 + * characters. Changing that would change every value the consumer already stores, so it stays.) * * Standard HTML and schema.org only — nothing here may know about a particular site. */ @@ -79,219 +82,349 @@ export const PAGE_FACT_BOUNDS: PageFactBounds = { maxOfferField: 64, }; +// Refusal threshold, not a sample size: a variant-heavy PDP legitimately carries dozens of offer +// triples, but past this the page posts NO claim rather than a truncated one (see +// `extractPageClaims` — a deterministic sample can systematically disagree with the consumer's +// endpoint). +export const STRUCTURED_OFFER_CAP = 200; + /** - * Read the page's facts off the live DOM. Runs IN THE PAGE (passed to `page.evaluate`), so it is fully - * self-contained: no imports, no closure over module scope, every helper declared inside. + * Both claims, as one in-page extraction returns them. Each is `null` when its reader found nothing to + * claim or threw — independently: a throw in one reader never costs the other its value. + */ +export type PageClaims = { + /** See `RenderJob.structuredOffers`: sorted `[price, currency, availability]` triples, flattened. */ + structuredOffers: Array | null; + pageFacts: PageFacts | null; +}; + +/** + * Read both of the page's claims off the live DOM in ONE `page.evaluate`. Runs IN THE PAGE, so it is + * fully self-contained: no imports, no closure over module scope, every helper declared inside. + * + * ONE ROUND TRIP, ONE PARSE. The two readers search the same ``; +const rawLd = (text: string) => ``; +const doc = (head: string, body = '') => `${head}${body}`; +/** The structuredOffers suite's page shell. */ +const soPage = (head: string) => + `p + +${head}

product

`; + +const soffer = (i: number) => ({ price: `${i}.99`, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }); +const foffer = (i: number) => ({ sku: `S${i}`, price: i, priceCurrency: 'USD', availability: 'InStock' }); +const variant = (sku: string, offers: unknown, extra: Record = {}) => ({ + '@type': 'Product', + sku, + offers, + ...extra, +}); +const crumbs = (length: number) => + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': Array.from({ length }, (_, i) => ({ position: i + 1, name: `C${i + 1}` })), + }); + +// ── the structuredOffers suite's fixtures ────────────────────────────────────────────────────── + +const STRUCTURED_OFFERS_FIXTURES: Array<[string, string]> = [ + [ + 'offers: two offers, number and string price', + soPage( + ld({ + '@type': 'Product', + 'name': 'Thing', + 'offers': [ + { price: 35.99, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { price: '29.99', priceCurrency: 'USD', availability: 'OutOfStock' }, + ], + }) + ), + ], + [ + 'offers: @graph with a single offer', + soPage( + ld({ + '@graph': [ + { '@type': 'BreadcrumbList' }, + { + '@type': ['Product'], + 'offers': { price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + }, + ], + }) + ), + ], + [ + 'offers: malformed block then a valid one', + soPage( + `${rawLd('{ this is not json ')}\n${ld({ '@type': 'Product', 'offers': { price: 5, availability: 'InStock' } })}` + ), + ], + [ + 'offers: trailing slash on availability', + soPage( + ld({ + '@type': 'Product', + 'offers': { price: 7.5, priceCurrency: 'USD', availability: 'https://schema.org/InStock/' }, + }) + ), + ], + ['offers: none (WebPage only)', soPage(ld({ '@type': 'WebPage' }))], + [ + 'offers: 250 products in one @graph (over the cap)', + soPage(ld({ '@graph': Array.from({ length: 250 }, (_, i) => ({ '@type': 'Product', 'offers': soffer(i) })) })), + ], + [ + 'offers: a 100k-character price is sliced', + soPage( + ld({ '@type': 'Product', 'offers': { price: 'x'.repeat(100000), priceCurrency: 'USD', availability: 'InStock' } }) + ), + ], +]; + +// ── the pageFacts suite's fixtures ───────────────────────────────────────────────────────────── + +const over = (n: number, c = 'x') => c.repeat(n + 1); +const PAGE_FACTS_FIXTURES: Array<[string, string]> = [ + [ + 'facts: head facts', + doc( + ` + Blue Widget | Example Store + + + + `, + `

+ Blue + Widget \t2000 +

second h1 is ignored

` + ), + ], + ['facts: empty document', doc('')], + [ + 'facts: empty values', + doc(' ', '

\n

'), + ], + [ + 'facts: Product with brand object, image array, rating, per-SKU offers', + doc( + ld({ + '@context': 'https://schema.org', + '@type': 'Product', + 'name': 'Blue Widget', + 'brand': { '@type': 'Brand', 'name': 'Example Brand' }, + 'image': ['https://www.example.com/img/1.jpg', 'https://www.example.com/img/2.jpg'], + 'aggregateRating': { '@type': 'AggregateRating', 'ratingValue': 4.6, 'ratingCount': 212, 'reviewCount': 90 }, + 'offers': [ + { sku: 'SKU-B', price: 35.99, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { sku: 1234567, price: '29.99', priceCurrency: 'USD', availability: 'http://schema.org/OutOfStock/' }, + { sku: 'SKU-A', price: 0, priceCurrency: 'USD', availability: 'PreOrder' }, + ], + }) + ), + ], + [ + 'facts: ProductGroup inside @graph with an AggregateOffer', + doc( + ld({ + '@context': 'https://schema.org', + '@graph': [ + { '@type': 'WebPage', 'name': 'not the product' }, + { + '@type': 'ProductGroup', + 'name': 'Widget Family', + 'brand': 'Example Brand', + 'image': { '@type': 'ImageObject', 'url': 'https://www.example.com/img/family.jpg' }, + 'aggregateRating': { ratingValue: '4.2', reviewCount: '37' }, + 'offers': { + '@type': 'AggregateOffer', + 'lowPrice': 10, + 'highPrice': 20, + 'offers': [ + { sku: 'S', price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { sku: 'L', price: 20, priceCurrency: 'USD', availability: 'https://schema.org/LimitedAvailability' }, + ], + }, + }, + ], + }) + ), + ], + [ + 'facts: top-level array, @type array, first product wins', + doc( + ld([ + { '@type': 'WebSite', 'name': 'Example Store' }, + { '@type': ['Product', 'Thing'], 'name': 'First', 'image': 'https://www.example.com/first.jpg' }, + ]) + ld({ '@type': 'Product', 'name': 'Second' }) + ), + ], + [ + 'facts: malformed, null and string blocks beside valid ones', + doc( + rawLd('{ "@type": "Product", "name": broken ') + + ld({ '@type': 'Product', 'name': 'Survivor', 'offers': { sku: 'X', price: 5, availability: 'InStock' } }) + + rawLd('null') + + rawLd('"just a string"') + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }) + ), + ], + [ + 'facts: a Product that states nothing else', + doc( + ld({ + '@type': 'Product', + 'brand': { '@type': 'Brand' }, + 'image': [], + 'aggregateRating': { ratingValue: 'n/a', ratingCount: null }, + 'offers': { '@type': 'AggregateOffer', 'lowPrice': 1, 'highPrice': 9 }, + }) + ), + ], + ['facts: rating value only', doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: 5 } }))], + [ + 'facts: rating null count falls back to reviewCount', + doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: '3.5', ratingCount: null, reviewCount: 8 } })), + ], + ['facts: rating zero', doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: 0, ratingCount: 0 } }))], + [ + 'facts: odd offer field types', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { price: 12 }, + { sku: { nested: true }, price: true, priceCurrency: 840, availability: 42 }, + 'not an offer', + null, + ], + }) + ), + ], + [ + 'facts: breadcrumb ordering', + doc( + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': [ + { '@type': 'ListItem', 'position': 3, 'item': { '@id': 'https://www.example.com/c/w', 'name': 'Widgets' } }, + { '@type': 'ListItem', 'position': '1', 'name': 'Home', 'item': 'https://www.example.com/' }, + { '@type': 'ListItem', 'name': 'Unpositioned' }, + { '@type': 'ListItem', 'position': 2, 'name': 'Shop', 'item': { name: '' } }, + { '@type': 'ListItem', 'position': 4 }, + { '@type': 'ListItem', 'position': 5, 'name': 'Element name', 'item': { name: 'Item name' } }, + ], + }) + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'second list is ignored' }] }) + ), + ], + [ + 'facts: breadcrumbs with no named crumbs', + doc(ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1 }] })), + ], + [ + 'facts: strings past and at maxString', + doc( + `${over(PAGE_FACT_BOUNDS.maxString, 'b')} + ` + + ld({ + '@type': 'Product', + 'name': over(PAGE_FACT_BOUNDS.maxString, 'b'), + 'brand': 'a'.repeat(PAGE_FACT_BOUNDS.maxString), + 'image': over(PAGE_FACT_BOUNDS.maxString, 'b'), + }), + `

${over(PAGE_FACT_BOUNDS.maxString, 'b')}

` + ), + ], + [ + 'facts: exactly maxOffers offers', + doc(ld({ '@type': 'Product', 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => foffer(i)) })), + ], + [ + 'facts: maxOffers + 1 across an AggregateOffer', + doc( + ld({ + '@type': 'Product', + 'name': 'Kept', + 'offers': [ + foffer(-1), + { + '@type': 'AggregateOffer', + 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => foffer(i)), + }, + ], + }) + ), + ], + ['facts: exactly maxBreadcrumbs', doc(crumbs(PAGE_FACT_BOUNDS.maxBreadcrumbs))], + ['facts: maxBreadcrumbs + 1', doc(crumbs(PAGE_FACT_BOUNDS.maxBreadcrumbs + 1))], + [ + 'facts: one overlong breadcrumb name', + doc( + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': [ + { position: 1, name: 'Home' }, + { position: 2, name: over(PAGE_FACT_BOUNDS.maxString) }, + ], + }) + ), + ], + [ + 'facts: offer fields past and at maxOfferField', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { + sku: over(PAGE_FACT_BOUNDS.maxOfferField), + price: over(PAGE_FACT_BOUNDS.maxOfferField), + priceCurrency: over(PAGE_FACT_BOUNDS.maxOfferField), + availability: `https://schema.org/${over(PAGE_FACT_BOUNDS.maxOfferField)}`, + }, + { + sku: 'k'.repeat(PAGE_FACT_BOUNDS.maxOfferField), + price: '1.00', + priceCurrency: 'USD', + availability: 'https://schema.org/InStock', + }, + ], + }) + ), + ], + [ + 'facts: one sku-less offer takes the product sku', + doc( + ld({ + '@type': 'Product', + 'sku': 9001, + 'offers': { price: 5, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + }) + ), + ], + [ + 'facts: one aggregated offer with an empty sku', + doc( + ld({ + '@type': 'Product', + 'sku': 'P-1', + 'offers': { '@type': 'AggregateOffer', 'offers': [{ price: 5, priceCurrency: 'USD', sku: '' }] }, + }) + ), + ], + ['facts: the offer sku wins', doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': { sku: 'O-1', price: 5 } }))], + [ + 'facts: several offers never take the product sku', + doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': [{ price: 5 }, { sku: 'O-2', price: 6 }] })), + ], + [ + 'facts: a refused offer sku is not replaced', + doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': { sku: over(PAGE_FACT_BOUNDS.maxOfferField), price: 5 } })), + ], + [ + 'facts: an overlong inherited sku', + doc(ld({ '@type': 'Product', 'sku': over(PAGE_FACT_BOUNDS.maxOfferField, 'y'), 'offers': { price: 5 } })), + ], + [ + 'facts: ProductGroup reads its hasVariant offers', + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Widget Family', + 'productGroupID': 'FAM-1', + 'hasVariant': [ + variant( + 'V-S', + { price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { + name: 'Widget, small', + brand: { '@type': 'Brand', 'name': 'Variant Brand' }, + image: 'https://www.example.com/img/small.jpg', + aggregateRating: { ratingValue: 4, ratingCount: 3 }, + } + ), + variant('V-L', [ + { price: 20, priceCurrency: 'USD', availability: 'https://schema.org/OutOfStock' }, + { sku: 'V-L-REFURB', price: 15, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + ]), + 'not a variant', + ], + }) + ), + ], + [ + "facts: a ProductGroup's own offers and fields win", + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Group name', + 'brand': 'Group Brand', + 'image': 'https://www.example.com/img/group.jpg', + 'aggregateRating': { ratingValue: 5, reviewCount: 1 }, + 'offers': { '@type': 'AggregateOffer', 'offers': [{ sku: 'G-1', price: 1 }] }, + 'hasVariant': [variant('V-1', { price: 2 }, { name: 'Variant name', brand: 'Variant Brand' })], + }) + ), + ], + [ + 'facts: a ProductGroup value past its bound', + doc( + ld({ + '@type': 'ProductGroup', + 'name': over(PAGE_FACT_BOUNDS.maxString, 'n'), + 'hasVariant': [variant('V-1', { price: 2 }, { name: 'Variant name' })], + }) + ), + ], + [ + 'facts: variant offers over the cap together', + doc( + ld({ + '@type': 'ProductGroup', + 'hasVariant': [ + variant( + 'A', + Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => ({ price: i })) + ), + variant('B', [{ price: 0 }]), + ], + }) + ), + ], + [ + 'facts: a refused group never falls through', + doc( + ld({ + '@type': 'ProductGroup', + 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers + 1 }, (_, i) => ({ price: i })), + 'hasVariant': [variant('A', { price: 1 })], + }) + ), + ], + [ + 'facts: mainEntity object', + doc( + ld({ + '@type': 'WebPage', + 'mainEntity': { '@type': 'Product', 'name': 'Nested', 'offers': { sku: 'N-1', price: 3 } }, + }) + ), + ], + [ + 'facts: mainEntity array inside @graph', + doc( + ld({ + '@context': 'https://schema.org', + '@graph': [ + { + '@type': 'ItemPage', + 'mainEntity': [ + { '@type': 'Organization', 'name': 'not it' }, + { '@type': 'ProductGroup', 'name': 'Nested group' }, + ], + }, + ], + }) + ), + ], + [ + 'facts: mainEntityOfPage', + doc(ld({ '@type': 'WebPage', 'mainEntityOfPage': { '@type': 'Product', 'name': 'Of page' } })), + ], + [ + 'facts: nested breadcrumb', + doc( + ld({ + '@type': 'CollectionPage', + 'breadcrumb': { + '@type': 'BreadcrumbList', + 'itemListElement': [ + { position: 2, name: 'Widgets' }, + { position: 1, name: 'Home' }, + ], + }, + }) + ), + ], + [ + 'facts: top-level beats nested', + doc( + ld({ + '@type': 'WebPage', + 'mainEntity': { '@type': 'Product', 'name': 'Nested' }, + 'breadcrumb': { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Nested crumb' }] }, + }) + + ld({ '@type': 'Product', 'name': 'Top level' }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Top crumb' }] }) + ), + ], + [ + 'facts: one level deep, only under page nodes', + doc( + ld({ + '@graph': [ + { + '@type': 'WebPage', + 'mainEntity': { '@type': 'WebPage', 'mainEntity': { '@type': 'Product', 'name': 'Deep' } }, + }, + { + '@type': 'Organization', + 'mainEntity': { '@type': 'Product', 'name': 'Under an organization' }, + 'breadcrumb': { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'no' }] }, + }, + { + '@type': 'WebPage', + 'mainEntityOfPage': 'https://www.example.com/p', + 'breadcrumb': { itemListElement: [{ position: 1, name: 'untyped' }] }, + }, + ], + }) + ), + ], +]; + +// ── new edge cases: where the two readers' walks DIVERGE, so the shared parse must serve both ── + +const EDGE_FIXTURES: Array<[string, string]> = [ + [ + // pageFacts stops after block 0 (product + breadcrumbs found); structuredOffers must still read the + // later blocks — parsed only for it. + 'edge: facts stop early, offers read on', + doc( + ld({ + '@graph': [ + { '@type': 'Product', 'name': 'First', 'offers': { sku: 'A', price: 1, priceCurrency: 'USD' } }, + { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }, + ], + }) + + rawLd('{ broken') + + ld({ '@type': 'Product', 'name': 'Second', 'offers': [{ price: 2 }, { price: 3, availability: 'InStock' }] }) + + ld([{ '@type': 'Product', 'offers': { price: '0.50', priceCurrency: 'EUR' } }]) + ), + ], + [ + // structuredOffers overflows in block 1; pageFacts still needs block 3 for its breadcrumbs. + 'edge: offers overflow early, facts read on', + doc( + ld({ '@type': 'Organization', 'name': 'Example' }) + + ld({ + '@graph': Array.from({ length: STRUCTURED_OFFER_CAP + 1 }, (_, i) => ({ + '@type': 'Product', + 'offers': soffer(i), + })), + }) + + ld({ '@type': 'WebSite' }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Late crumb' }] }) + ), + ], + [ + 'edge: exactly the offer cap across two products', + doc( + ld({ + '@graph': [ + { '@type': 'Product', 'offers': Array.from({ length: 150 }, (_, i) => soffer(i)) }, + { + '@type': 'Product', + 'offers': Array.from({ length: STRUCTURED_OFFER_CAP - 150 }, (_, i) => soffer(i + 150)), + }, + ], + }) + ), + ], + [ + 'edge: one offer past the cap across two blocks', + doc( + ld({ '@type': 'Product', 'offers': Array.from({ length: STRUCTURED_OFFER_CAP }, (_, i) => soffer(i)) }) + + ld({ '@type': 'Product', 'offers': soffer(999) }) + ), + ], + [ + // A ProductGroup is not a `Product` to structuredOffers (null), while pageFacts reads its variants. + 'edge: ProductGroup — offers see nothing, facts see the variants', + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Family', + 'hasVariant': [variant('V1', { price: 4, priceCurrency: 'USD', availability: 'https://schema.org/InStock' })], + }) + ), + ], + [ + 'edge: JSON primitives, empty and whitespace blocks', + doc( + rawLd('') + + rawLd(' \n ') + + rawLd('42') + + rawLd('true') + + rawLd('[]') + + rawLd('[null, 1, "x", []]') + + rawLd('{"@graph": {"@type": "Product", "offers": {"price": 1}}}') + + ld({ '@type': 'Product', 'name': 'After the junk', 'offers': { price: 8 } }) + ), + ], + [ + 'edge: over-long strings everywhere', + doc( + `${'t'.repeat(5000)}` + + ld({ + '@type': 'Product', + 'name': 'n'.repeat(3000), + 'sku': 's'.repeat(65), + 'offers': [ + { + price: 'p'.repeat(100000), + priceCurrency: 'c'.repeat(70), + availability: `https://schema.org/${'a'.repeat(80)}`, + }, + { price: 1, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + ], + }), + `

${'h '.repeat(1500)}

` + ), + ], + [ + 'edge: sort order with nulls, duplicates and non-ASCII', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { price: 'Ü', priceCurrency: 'USD' }, + { price: 'Z' }, + { availability: 'InStock' }, + { price: 'Z' }, + { price: '10', priceCurrency: 'USD', availability: 'https://schema.org/OutOfStock' }, + { price: '9', priceCurrency: 'usd', availability: '/' }, + { price: 1e21, priceCurrency: '€', availability: 'https://schema.org/' }, + { price: -0, priceCurrency: 'USD', availability: '' }, + ], + 'aggregateRating': { ratingValue: -0, ratingCount: ' 12 ' }, + }) + ), + ], + [ + 'edge: many small blocks', + doc( + Array.from({ length: 60 }, (_, i) => + i % 7 === 3 + ? rawLd('{ nope') + : ld({ '@type': i % 5 ? 'Thing' : 'Product', 'name': `N${i}`, 'offers': { price: i } }) + ).join('') + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Last' }] }) + ), + ], +]; + +// ── pages that break one reader: each side must null exactly what it nulled before ───────────── + +const hostile = (script: string) => + doc( + `Hostile` + + ld({ '@type': 'Product', 'name': 'H', 'offers': { sku: 'H1', price: 2, priceCurrency: 'USD' } }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }), + `

Hostile

` + ); +const HOSTILE_FIXTURES: Array<[string, string, (c: Claims) => void]> = [ + [ + 'hostile: document.title throws — facts null, offers kept', + hostile(`Object.defineProperty(document, 'title', { get() { throw new Error('hostile'); } });`), + (c) => { + assert.equal(c.pageFacts, null); + assert.deepEqual(c.structuredOffers, ['2', 'USD', null]); + }, + ], + [ + 'hostile: document.querySelector throws — facts null, offers kept', + hostile(`document.querySelector = () => { throw new Error('hostile'); };`), + (c) => { + assert.equal(c.pageFacts, null); + assert.deepEqual(c.structuredOffers, ['2', 'USD', null]); + }, + ], + [ + 'hostile: Array.prototype.flat throws — offers null, facts kept', + hostile(`Array.prototype.flat = function () { throw new Error('hostile'); };`), + (c) => { + assert.equal(c.structuredOffers, null); + assert.equal(c.pageFacts?.product?.name, 'H'); + assert.deepEqual(c.pageFacts?.breadcrumbs, ['Home']); + }, + ], + [ + 'hostile: document.querySelectorAll throws — both null', + hostile(`document.querySelectorAll = () => { throw new Error('hostile'); };`), + (c) => assert.deepEqual(c, { structuredOffers: null, pageFacts: null }), + ], + [ + 'hostile: one block’s textContent throws — that block is skipped by both', + hostile( + `Object.defineProperty(document.querySelector('script[type="application/ld+json"]'), 'textContent', { get() { throw new Error('hostile'); } });` + ), + (c) => { + assert.equal(c.structuredOffers, null, 'the only Product block was unreadable'); + assert.equal(c.pageFacts?.product, null); + assert.deepEqual(c.pageFacts?.breadcrumbs, ['Home']); + }, + ], +]; + +test('every structuredOffers-suite fixture: merged output is identical to the 1.37.0 pair', async () => { + for (const [label, html] of STRUCTURED_OFFERS_FIXTURES) await compare(label, html); +}); + +test('every pageFacts-suite fixture: merged output is identical to the 1.37.0 pair', async () => { + for (const [label, html] of PAGE_FACTS_FIXTURES) await compare(label, html); +}); + +test('edge cases where the two walks diverge: merged output is identical to the 1.37.0 pair', async () => { + const results = new Map(); + for (const [label, html] of EDGE_FIXTURES) results.set(label, await compare(label, html)); + // Not vacuous: the divergent walks really did produce both claims. + const early = results.get('edge: facts stop early, offers read on')!; + assert.equal(early.structuredOffers?.length, 4 * 3, 'offers from blocks past the one facts stopped at'); + assert.deepEqual(early.pageFacts?.breadcrumbs, ['Home']); + const late = results.get('edge: offers overflow early, facts read on')!; + assert.equal(late.structuredOffers, null); + assert.deepEqual(late.pageFacts?.breadcrumbs, ['Late crumb']); +}); + +test('a page that breaks one reader: each claim is null exactly where the 1.37.0 pair made it null', async () => { + for (const [label, html, check] of HOSTILE_FIXTURES) check(await compare(label, html)); +}); + +// ── seeded fuzz ──────────────────────────────────────────────────────────────────────────────── + +/** mulberry32 — a fixed seed, so a failure reproduces. */ +const prng = (seed: number) => () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +}; + +const fuzzDocuments = (count: number, seed: number): string[] => { + const rand = prng(seed); + const pick = (xs: readonly T[]): T => xs[Math.floor(rand() * xs.length)]; + const maybe = (p: number) => rand() < p; + const STRINGS = [ + 'Widget', + '', + ' padded ', + 'Ünïcødé ✓', + 'x'.repeat(64), + 'x'.repeat(65), + 'y'.repeat(PAGE_FACT_BOUNDS.maxString), + 'z'.repeat(PAGE_FACT_BOUNDS.maxString + 1), + 'a/b/', + '35.99', + ' 4.5 ', + 'n/a', + ]; + const SCALARS = [0, -0, 12.5, 1e21, -3, 1234567, true, false, null, {}, [], { name: 'obj' }]; + const TYPES = [ + 'Product', + 'Product', + 'ProductGroup', + 'BreadcrumbList', + 'WebPage', + 'ItemPage', + 'CollectionPage', + 'Organization', + 'Offer', + 'AggregateOffer', + 'Thing', + ['Product', 'Thing'], + ['ProductGroup'], + ['WebPage', 'ItemPage'], + undefined, + 42, + ]; + const AVAILABILITY = [ + 'https://schema.org/InStock', + 'http://schema.org/OutOfStock/', + 'InStock', + 'PreOrder', + '', + '/', + 'https://schema.org/', + 42, + null, + 'x'.repeat(70), + `https://schema.org/${'x'.repeat(70)}`, + ]; + const value = () => (maybe(0.6) ? pick(STRINGS) : pick(SCALARS)); + const offer = (nested = false): unknown => { + if (maybe(0.05)) return pick(['not an offer', null, 7]); + const o: Record = {}; + if (maybe(0.3)) o['@type'] = pick(['Offer', 'AggregateOffer']); + if (maybe(0.85)) o.price = maybe(0.6) ? pick([9.99, 0, -0, '35.99', 1e21, 100, '1,299.00']) : value(); + if (maybe(0.7)) o.priceCurrency = maybe(0.8) ? pick(['USD', 'EUR', 'usd', '€']) : value(); + if (maybe(0.7)) o.availability = pick(AVAILABILITY); + if (maybe(0.5)) o.sku = maybe(0.7) ? `SKU-${Math.floor(rand() * 50)}` : value(); + if (!nested && o['@type'] === 'AggregateOffer' && maybe(0.7)) o.offers = offers(true); + return o; + }; + const offerCount = () => pick([0, 1, 1, 1, 2, 2, 3, 5, 8, 199, 200, 201]); + const offers = (nested = false): unknown => { + const list = Array.from({ length: offerCount() }, () => offer(nested)); + switch (pick(['array', 'array', 'single', 'aggregate', 'mixed', 'absent'])) { + case 'single': + return list[0] ?? offer(nested); + case 'aggregate': + return { '@type': 'AggregateOffer', 'lowPrice': 1, 'highPrice': 9, 'offers': maybe(0.8) ? list : undefined }; + case 'mixed': + return [offer(nested), { '@type': 'AggregateOffer', 'offers': list }, ...list.slice(0, 2)]; + case 'absent': + return pick([undefined, null, '', 'string offers']); + default: + return list; + } + }; + const crumbList = (): unknown => { + const n = pick([0, 1, 2, 3, 5, 30, 31]); + return { + '@type': maybe(0.9) ? 'BreadcrumbList' : undefined, + 'itemListElement': Array.from({ length: n }, (_, i) => + maybe(0.05) + ? 'junk' + : { + '@type': 'ListItem', + 'position': maybe(0.8) ? pick([i + 1, String(i + 1), n - i, 1, null, 'x']) : undefined, + 'name': maybe(0.8) ? pick([`C${i}`, '', 'Home', 'z'.repeat(PAGE_FACT_BOUNDS.maxString + 1)]) : undefined, + 'item': maybe(0.4) ? pick([{ name: `Item ${i}` }, { name: '' }, 'https://www.example.com/']) : undefined, + } + ), + }; + }; + const node = (depth: number): unknown => { + if (maybe(0.04)) return pick([null, 'text', 3, []]); + const type = pick(TYPES); + const n: Record = { '@type': type }; + if (maybe(0.7)) n.name = value(); + if (maybe(0.4)) n.brand = maybe(0.5) ? { '@type': 'Brand', 'name': value() } : value(); + if (maybe(0.4)) + n.image = pick([ + value(), + [value(), 'second'], + { '@type': 'ImageObject', 'url': value() }, + [], + [{ url: 'https://www.example.com/i.jpg' }], + ]); + if (maybe(0.4)) + n.aggregateRating = { ratingValue: value(), ratingCount: maybe(0.5) ? value() : undefined, reviewCount: value() }; + if (maybe(0.5)) n.sku = maybe(0.6) ? `P-${Math.floor(rand() * 20)}` : value(); + if (maybe(0.7)) n.offers = offers(); + if (maybe(0.2) || type === 'ProductGroup') + n.hasVariant = Array.from({ length: pick([0, 1, 2, 3]) }, () => node(depth + 1)); + if (type === 'BreadcrumbList' || maybe(0.1)) + n.itemListElement = (crumbList() as Record).itemListElement; + if (depth < 2 && maybe(0.25)) n.mainEntity = maybe(0.5) ? node(depth + 1) : [node(depth + 1), node(depth + 1)]; + if (depth < 2 && maybe(0.15)) n.mainEntityOfPage = maybe(0.6) ? node(depth + 1) : 'https://www.example.com/p'; + if (depth < 2 && maybe(0.2)) n.breadcrumb = crumbList(); + return n; + }; + const block = (): string => { + switch (pick(['node', 'node', 'node', 'array', 'graph', 'graph', 'malformed', 'primitive', 'graph-object'])) { + case 'array': + return ld(Array.from({ length: pick([0, 1, 2, 4]) }, () => node(0))); + case 'graph': + return ld({ + '@context': 'https://schema.org', + '@graph': Array.from({ length: pick([1, 2, 3, 6]) }, () => node(0)), + }); + case 'graph-object': + return ld({ '@graph': node(0) }); + case 'malformed': + return rawLd(pick(['{ "@type": "Product", ', '', ' ', '{nope}', '[1,'])); + case 'primitive': + return rawLd(pick(['null', '"str"', '42', 'true', '[]'])); + default: + return ld(node(0)); + } + }; + const text = () => + pick(['Blue Widget', '', ' spaced out ', 'w'.repeat(PAGE_FACT_BOUNDS.maxString + 1), 'Ünïcødé']); + return Array.from({ length: count }, () => { + const head = [ + maybe(0.8) ? `${text()}` : '', + maybe(0.6) ? `` : '', + maybe(0.6) + ? `` + : '', + ...Array.from({ length: pick([0, 1, 1, 2, 3, 5]) }, block), + ].join(''); + const body = maybe(0.7) ? `

${text()} ${text()}

` : ''; + return `${head}${body}`; + }); +}; + +test('seeded fuzz: 400 generated documents, merged output identical to the 1.37.0 pair on every one', async () => { + const documents = fuzzDocuments(400, 0x5eed138); + const page = await browser.newPage(); + const seen = { offers: 0, offersNull: 0, product: 0, productOffers: 0, breadcrumbs: 0, capRefusal: 0 }; + try { + pages.set('/fuzz', doc('')); + await page.goto(`${base}/fuzz`, { waitUntil: 'load' }); + for (let i = 0; i < documents.length; i++) { + await page.evaluate((html) => { + document.documentElement.innerHTML = html; + }, documents[i]); + const old = await before137(page); + const merged = await after138(page); + assertParity(old, merged, `fuzz #${i}`); + if (merged.structuredOffers) seen.offers++; + else seen.offersNull++; + if (merged.pageFacts?.product) seen.product++; + if (merged.pageFacts?.product?.offers) seen.productOffers++; + if (merged.pageFacts?.breadcrumbs) seen.breadcrumbs++; + if (documents[i].split('"price"').length > STRUCTURED_OFFER_CAP && !merged.structuredOffers) seen.capRefusal++; + } + } finally { + await page.close(); + } + // The corpus must actually exercise both readers, not compare a wall of nulls. + for (const [what, n] of Object.entries(seen)) assert.ok(n >= 20, `fuzz coverage: only ${n} documents with ${what}`); +}); + +// ── the 1.37.0 extractors, VERBATIM (renamed only) ───────────────────────────────────────────── +// `extractStructuredOffers` from src/renderer.ts and `extractPageFacts` from src/pageFacts.ts at +// browser 1.37.0. Do not edit: they are the reference the merged extractor is held to. + +function legacyStructuredOffers(cap: number): Array | null { + const triples: Array> = []; + let overflowed = false; + // Field-level byte bound: the cap bounds triple COUNT, so without this a single pathological + // field (a megabyte "price" string) would still inflate every posted result for the page. No + // legitimate price, currency code, or availability token approaches 64 characters. + const bound = (value: string): string | null => (value === '' ? null : value.slice(0, 64)); + const collect = (node: unknown) => { + if (overflowed || !node || typeof node !== 'object') return; + const record = node as Record; + const type = record['@type']; + const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product')); + if (!isProduct) return; + const raw = record.offers; + const list = Array.isArray(raw) ? raw : raw ? [raw] : []; + for (const entry of list) { + if (!entry || typeof entry !== 'object') continue; + if (triples.length >= cap) { + overflowed = true; + return; + } + const offer = entry as Record; + // filter(Boolean) before pop: a trailing slash (https://schema.org/InStock/) would + // otherwise pop the empty segment and read as no availability at all. + const availability = + typeof offer.availability === 'string' + ? bound(offer.availability.split('/').filter(Boolean).pop() ?? '') + : null; + const price = offer.price === undefined || offer.price === null ? null : bound(String(offer.price)); + const currency = typeof offer.priceCurrency === 'string' ? bound(offer.priceCurrency) : null; + triples.push([price, currency, availability]); + } + }; + document.querySelectorAll('script[type="application/ld+json"]').forEach((el) => { + if (overflowed) return; + let data: unknown; + try { + data = JSON.parse(el.textContent || ''); + } catch { + return; // one malformed block must not cost the page its other blocks + } + const graph = (data as Record)?.['@graph']; + const nodes = Array.isArray(data) ? data : Array.isArray(graph) ? graph : [data]; + for (const node of nodes) { + if (overflowed) break; + collect(node); + } + }); + if (overflowed || !triples.length) return null; + // Field-wise, not JSON.stringify per comparison: sorting is O(n log n) COMPARISONS, so + // stringifying inside the comparator serialises every triple many times over. Code-unit + // comparison, not localeCompare: the whole point of the sort is a sequence that is identical + // across renders, and collation varies with the browser's locale/ICU. + triples.sort((a, b) => { + for (let i = 0; i < 3; i++) { + const x = a[i]; + const y = b[i]; + if (x === y) continue; + if (x === null) return -1; + if (y === null) return 1; + return x < y ? -1 : 1; + } + return 0; + }); + return triples.flat(); +} + +function legacyPageFacts(bounds: PageFactBounds): PageFacts { + type Json = Record; + const isObject = (value: unknown): value is Json => !!value && typeof value === 'object' && !Array.isArray(value); + const boundTo = + (max: number) => + (value: unknown): string | null => + typeof value !== 'string' || value === '' || value.length > max ? null : value; + const str = boundTo(bounds.maxString); + const offerField = boundTo(bounds.maxOfferField); + const hasType = (node: Json, wanted: string[]) => { + const type = node['@type']; + return Array.isArray(type) ? type.some((t) => wanted.includes(t as string)) : wanted.includes(type as string); + }; + // A schema.org number may be written as a JSON number or a numeric string ("4.5"). Anything else — + // including null, which Number() would silently read as 0 — is no claim. + const num = (value: unknown): number | null => { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!trimmed || trimmed.length > bounds.maxOfferField) return null; + const n = Number(trimmed); + return Number.isFinite(n) ? n : null; + }; + const listOf = (value: unknown): unknown[] => (Array.isArray(value) ? value : value ? [value] : []); + + const canonicalEl = document.querySelector('link[rel="canonical"]') as HTMLLinkElement | null; + const descriptionEl = document.querySelector('meta[name="description" i]'); + const h1El = document.querySelector('h1'); + + // schema.org WebPage and its subtypes: the nodes whose `mainEntity` / `breadcrumb` are searched. + const PAGE_TYPES = [ + 'WebPage', + 'ItemPage', + 'CollectionPage', + 'SearchResultsPage', + 'ProfilePage', + 'AboutPage', + 'ContactPage', + 'FAQPage', + 'QAPage', + 'CheckoutPage', + 'MedicalWebPage', + 'RealEstateListing', + 'MediaGallery', + 'ImageGallery', + 'VideoGallery', + ]; + const PRODUCT_TYPES = ['Product', 'ProductGroup']; + + // A node at the top of a block (or of its @graph) always wins; one found a single level down — a + // page node's `mainEntity` / `mainEntityOfPage` / `breadcrumb` — is used only when no top-level node + // matched. One level only: this reads what a page states about itself, it does not crawl a graph. + let productNode: Json | null = null; + let breadcrumbNode: Json | null = null; + let nestedProduct: Json | null = null; + let nestedBreadcrumb: Json | null = null; + const blocks = document.querySelectorAll('script[type="application/ld+json"]'); + for (let i = 0; i < blocks.length && !(productNode && breadcrumbNode); i++) { + let data: unknown; + try { + data = JSON.parse(blocks[i].textContent || ''); + } catch { + continue; // one malformed block must not cost the page its other blocks + } + const graph = isObject(data) ? data['@graph'] : undefined; + const nodes = Array.isArray(data) ? data : Array.isArray(graph) ? graph : [data]; + for (const node of nodes) { + if (!isObject(node)) continue; + if (!productNode && hasType(node, PRODUCT_TYPES)) productNode = node; + if (!breadcrumbNode && hasType(node, ['BreadcrumbList'])) breadcrumbNode = node; + if (!hasType(node, PAGE_TYPES)) continue; + if (!nestedProduct) { + for (const entity of listOf(node.mainEntity)) { + if (isObject(entity) && hasType(entity, PRODUCT_TYPES)) { + nestedProduct = entity; + break; + } + } + } + const ofPage = node.mainEntityOfPage; + if (!nestedProduct && isObject(ofPage) && hasType(ofPage, PRODUCT_TYPES)) nestedProduct = ofPage; + const crumb = node.breadcrumb; + if (!nestedBreadcrumb && isObject(crumb) && hasType(crumb, ['BreadcrumbList'])) nestedBreadcrumb = crumb; + } + } + productNode = productNode ?? nestedProduct; + breadcrumbNode = breadcrumbNode ?? nestedBreadcrumb; + + // A JSON number is stated as its string form ("35.99", "12345"), as structuredOffers states a price. + const asString = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? String(value) : value); + const absent = (value: unknown) => value === undefined || value === null || value === ''; + const nonEmpty = (value: unknown) => (typeof value === 'string' && value !== '' ? value : undefined); + + // The offer objects a node states, in document order — an AggregateOffer summarises + // (lowPrice/highPrice), so only the offers it lists are offers. null when past the bound. + const offersOf = (raw: unknown): Json[] | null => { + const found: Json[] = []; + for (const entry of listOf(raw)) { + const list = isObject(entry) && hasType(entry, ['AggregateOffer']) ? listOf(entry.offers) : [entry]; + for (const offer of list) { + if (!isObject(offer)) continue; + if (found.length >= bounds.maxOffers) return null; + found.push(offer); + } + } + return found; + }; + const tuple = (offer: Json, sku: unknown): PageFactOffer => { + // Reduced exactly as extractStructuredOffers reduces it: the last non-empty path segment, so a + // trailing slash (https://schema.org/InStock/) still yields the verdict. + const availability = + typeof offer.availability === 'string' ? (offer.availability.split('/').filter(Boolean).pop() ?? '') : null; + return [ + offerField(asString(sku)), + offerField(asString(offer.price)), + offerField(offer.priceCurrency), + offerField(availability), + ]; + }; + + const readOffers = (node: Json, variants: Json[]): PageFactOffer[] | null => { + const own = offersOf(node.offers); + if (own === null) return null; // refused — never fall through to the variants + if (own.length) { + // A product selling ONE offer often names the SKU on itself rather than on the offer. Only + // then: with several offers, the product's SKU names none of them. + const inherit = own.length === 1 && absent(own[0].sku); + return own.map((offer) => tuple(offer, inherit ? node.sku : offer.sku)); + } + // A ProductGroup usually sells through its variants (`hasVariant`, each a Product with its own + // offers and SKU) — read those when the group itself states no offers. + const out: PageFactOffer[] = []; + for (const variant of variants) { + const offers = offersOf(variant.offers); + if (offers === null) return null; + for (const offer of offers) { + if (out.length >= bounds.maxOffers) return null; + out.push(tuple(offer, absent(offer.sku) ? variant.sku : offer.sku)); + } + } + return out.length ? out : null; + }; + + // Each field is PICKED raw — from the node, else (for a ProductGroup) its first variant — and only + // then bounded, so a value the group states too long is refused rather than swapped for a variant's. + const pickBrand = (n: Json) => nonEmpty(isObject(n.brand) ? n.brand.name : n.brand); + const pickImage = (n: Json) => { + const image = Array.isArray(n.image) ? n.image[0] : n.image; + return nonEmpty(isObject(image) ? image.url : image); + }; + const pickRating = (n: Json): [number | null, number | null] | undefined => { + const r = n.aggregateRating; + if (!isObject(r)) return undefined; + const pair: [number | null, number | null] = [num(r.ratingValue), num(r.ratingCount ?? r.reviewCount)]; + return pair[0] !== null || pair[1] !== null ? pair : undefined; + }; + + const readProduct = (node: Json): PageFactsProduct => { + const variants = hasType(node, ['ProductGroup']) ? listOf(node.hasVariant).filter(isObject) : []; + const pick = (from: (n: Json) => T | undefined): T | undefined => + from(node) ?? (variants.length ? from(variants[0]) : undefined); + return { + name: str(pick((n) => nonEmpty(n.name))), + brand: str(pick(pickBrand)), + image: str(pick(pickImage)), + rating: pick(pickRating) ?? null, + offers: readOffers(node, variants), + }; + }; + + const readBreadcrumbs = (node: Json): string[] | null => { + const crumbs: Array<{ name: string; position: number | null; order: number }> = []; + const elements = listOf(node.itemListElement); + for (let order = 0; order < elements.length; order++) { + const el = elements[order]; + if (!isObject(el)) continue; + const itemName = isObject(el.item) ? el.item.name : undefined; + const raw = typeof itemName === 'string' && itemName !== '' ? itemName : el.name; + if (typeof raw !== 'string' || raw === '') continue; // an unnamed crumb claims nothing + // A name past the bound refuses the whole trail: a trail with a hole in it would disagree. + if (raw.length > bounds.maxString) return null; + if (crumbs.length >= bounds.maxBreadcrumbs) return null; + crumbs.push({ name: raw, position: num(el.position), order }); + } + // Ordered by position; a crumb without one keeps its document order after the positioned ones. + crumbs.sort((a, b) => + a.position === b.position + ? a.order - b.order + : a.position === null + ? 1 + : b.position === null + ? -1 + : a.position - b.position + ); + return crumbs.length ? crumbs.map((c) => c.name) : null; + }; + + return { + canonical: canonicalEl ? str(canonicalEl.href) : null, + title: str((document.title || '').trim()), + metaDescription: descriptionEl ? str(descriptionEl.getAttribute('content')) : null, + h1: h1El ? str((h1El.textContent || '').replace(/\s+/g, ' ').trim()) : null, + product: productNode ? readProduct(productNode) : null, + breadcrumbs: breadcrumbNode ? readBreadcrumbs(breadcrumbNode) : null, + }; +} diff --git a/packages/browser/test/pageFacts.test.ts b/packages/browser/test/pageFacts.test.ts index df48eced..9b67a99b 100644 --- a/packages/browser/test/pageFacts.test.ts +++ b/packages/browser/test/pageFacts.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; import puppeteer, { type Browser } from 'puppeteer'; -import { extractPageFacts, PAGE_FACT_BOUNDS } from '../dist/pageFacts.js'; +import { extractPageClaims, PAGE_FACT_BOUNDS, STRUCTURED_OFFER_CAP } from '../dist/pageFacts.js'; import { renderOnce } from '../dist/renderOnce.js'; /** @@ -50,7 +50,7 @@ const factsOf = async (html: string) => { const page = await browser.newPage(); try { await page.goto(`${base}${path}`, { waitUntil: 'load' }); - return await page.evaluate(extractPageFacts, PAGE_FACT_BOUNDS); + return (await page.evaluate(extractPageClaims, STRUCTURED_OFFER_CAP, PAGE_FACT_BOUNDS)).pageFacts; } finally { await page.close(); } @@ -655,3 +655,43 @@ test('an extraction that throws posts pageFacts: null — present, never a faile assert.notEqual(result.job.pageFacts, undefined); assert.deepEqual(result.job.structuredOffers, ['9.5', 'USD', 'InStock']); }); + +test('an offers reader that throws posts structuredOffers: null — pageFacts still read, never a failed render', async () => { + // Both claims come from ONE evaluate now; each reader is caught inside the page, so a throw in the + // offers reader (here: the page breaks Array.prototype.flat, which only that reader calls) must cost + // the consumer the offers and nothing else — what two separate evaluates gave. + pages.set( + '/hostile-offers', + RENDERED(`${base}/hostile-offers`).replace( + '', + `` + ) + ); + const result = await renderOnce({ url: `${base}/hostile-offers`, config: NO_SCROLL }); + assert.equal(result.outcome, 'ok'); + assert.ok(result.html && result.html.length > 0); + assert.equal(result.job.structuredOffers, null); + assert.notEqual(result.job.structuredOffers, undefined); + assert.equal(result.job.pageFacts?.product?.name, 'Blue Widget'); + assert.deepEqual(result.job.pageFacts?.product?.offers, [['W1', '9.5', 'USD', 'InStock']]); +}); + +test('a failure of the extraction evaluate itself posts null for BOTH claims — present, never a failed render', async () => { + // No in-page catch can see a result that cannot be returned: the page makes the offers reader's + // output cyclic, so the evaluate itself rejects. Both claims are then null (ran, failed benignly), + // never absent ("renderer predates it"), and the render still succeeds. + pages.set( + '/unreturnable', + RENDERED(`${base}/unreturnable`).replace( + '', + `` + ) + ); + const result = await renderOnce({ url: `${base}/unreturnable`, config: NO_SCROLL }); + assert.equal(result.outcome, 'ok'); + assert.ok(result.html && result.html.length > 0); + assert.equal(result.job.structuredOffers, null); + assert.notEqual(result.job.structuredOffers, undefined); + assert.equal(result.job.pageFacts, null); + assert.notEqual(result.job.pageFacts, undefined); +});