diff --git a/package-lock.json b/package-lock.json index 7bb0c89d..07759f98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8516,7 +8516,7 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.37.0", + "version": "1.38.0", "license": "Apache-2.0", "dependencies": { "mqtt": "^5.10.4", diff --git a/packages/browser/README.md b/packages/browser/README.md index 35d4d0f0..5cce888e 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -142,7 +142,9 @@ carries facts the renderer read off the settled DOM (before `postProcess`), so t the cached page's own claims against its sources of truth without parsing HTML on its write path. Both fields share one wire contract: **absent** means the renderer predates the field (a consumer may alarm on it); **`null`** means the extraction ran and found nothing to claim or failed benignly — it is posted as -`null`, never omitted, and an extraction failure never fails the render. The extraction runs only on +`null`, never omitted, and an extraction failure never fails the render. Both are read in one in-page +pass (one `page.evaluate`, each JSON-LD block parsed once): a failure in one reader costs only that +field, and a failure of the pass itself posts `null` for both. The extraction runs only on the path that produces content, so a variant without content (a redirect, a verdict, an error) may carry neither — absence is meaningful only on a variant that carries content. diff --git a/packages/browser/package.json b/packages/browser/package.json index 0e169d35..116d24e8 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-browser", - "version": "1.37.0", + "version": "1.38.0", "type": "module", "description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.", "keywords": [ diff --git a/packages/browser/src/pageFacts.ts b/packages/browser/src/pageFacts.ts index 6ad358cb..95b8bea9 100644 --- a/packages/browser/src/pageFacts.ts +++ b/packages/browser/src/pageFacts.ts @@ -1,6 +1,7 @@ /** - * What the rendered page itself CLAIMS — its canonical, title, meta description, first heading, - * product JSON-LD and breadcrumb trail — read off the settled DOM and posted with each variant. + * What the rendered page itself CLAIMS — its schema.org offers (`structuredOffers`) and its SEO facts + * (canonical, title, meta description, first heading, product JSON-LD and breadcrumb trail) — read off + * the settled DOM and posted with each variant. * * WHY THE RENDERER. The consumer compares these against its own sources of truth to decide whether a * cached page is wrong (and only then re-render it). It deliberately never parses HTML on its hot @@ -10,7 +11,9 @@ * BOUNDS ARE REFUSALS, NEVER TRUNCATIONS. A truncated value would disagree with the consumer's source * of truth on every comparison, and a systematic disagreement means the consumer expires and * re-renders that page forever. So a value past its bound is reported as no claim at all (`null`), - * which degrades to exactly what "the page does not say" does: nothing to compare. + * which degrades to exactly what "the page does not say" does: nothing to compare. (`structuredOffers` + * predates this rule for its fields: it refuses past its offer cap but slices an over-long field to 64 + * characters. Changing that would change every value the consumer already stores, so it stays.) * * Standard HTML and schema.org only — nothing here may know about a particular site. */ @@ -79,219 +82,349 @@ export const PAGE_FACT_BOUNDS: PageFactBounds = { maxOfferField: 64, }; +// Refusal threshold, not a sample size: a variant-heavy PDP legitimately carries dozens of offer +// triples, but past this the page posts NO claim rather than a truncated one (see +// `extractPageClaims` — a deterministic sample can systematically disagree with the consumer's +// endpoint). +export const STRUCTURED_OFFER_CAP = 200; + /** - * Read the page's facts off the live DOM. Runs IN THE PAGE (passed to `page.evaluate`), so it is fully - * self-contained: no imports, no closure over module scope, every helper declared inside. + * Both claims, as one in-page extraction returns them. Each is `null` when its reader found nothing to + * claim or threw — independently: a throw in one reader never costs the other its value. + */ +export type PageClaims = { + /** See `RenderJob.structuredOffers`: sorted `[price, currency, availability]` triples, flattened. */ + structuredOffers: Array | null; + pageFacts: PageFacts | null; +}; + +/** + * Read both of the page's claims off the live DOM in ONE `page.evaluate`. Runs IN THE PAGE, so it is + * fully self-contained: no imports, no closure over module scope, every helper declared inside. + * + * ONE ROUND TRIP, ONE PARSE. The two readers search the same ``; +const rawLd = (text: string) => ``; +const doc = (head: string, body = '') => `${head}${body}`; +/** The structuredOffers suite's page shell. */ +const soPage = (head: string) => + `p + +${head}

product

`; + +const soffer = (i: number) => ({ price: `${i}.99`, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }); +const foffer = (i: number) => ({ sku: `S${i}`, price: i, priceCurrency: 'USD', availability: 'InStock' }); +const variant = (sku: string, offers: unknown, extra: Record = {}) => ({ + '@type': 'Product', + sku, + offers, + ...extra, +}); +const crumbs = (length: number) => + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': Array.from({ length }, (_, i) => ({ position: i + 1, name: `C${i + 1}` })), + }); + +// ── the structuredOffers suite's fixtures ────────────────────────────────────────────────────── + +const STRUCTURED_OFFERS_FIXTURES: Array<[string, string]> = [ + [ + 'offers: two offers, number and string price', + soPage( + ld({ + '@type': 'Product', + 'name': 'Thing', + 'offers': [ + { price: 35.99, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { price: '29.99', priceCurrency: 'USD', availability: 'OutOfStock' }, + ], + }) + ), + ], + [ + 'offers: @graph with a single offer', + soPage( + ld({ + '@graph': [ + { '@type': 'BreadcrumbList' }, + { + '@type': ['Product'], + 'offers': { price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + }, + ], + }) + ), + ], + [ + 'offers: malformed block then a valid one', + soPage( + `${rawLd('{ this is not json ')}\n${ld({ '@type': 'Product', 'offers': { price: 5, availability: 'InStock' } })}` + ), + ], + [ + 'offers: trailing slash on availability', + soPage( + ld({ + '@type': 'Product', + 'offers': { price: 7.5, priceCurrency: 'USD', availability: 'https://schema.org/InStock/' }, + }) + ), + ], + ['offers: none (WebPage only)', soPage(ld({ '@type': 'WebPage' }))], + [ + 'offers: 250 products in one @graph (over the cap)', + soPage(ld({ '@graph': Array.from({ length: 250 }, (_, i) => ({ '@type': 'Product', 'offers': soffer(i) })) })), + ], + [ + 'offers: a 100k-character price is sliced', + soPage( + ld({ '@type': 'Product', 'offers': { price: 'x'.repeat(100000), priceCurrency: 'USD', availability: 'InStock' } }) + ), + ], +]; + +// ── the pageFacts suite's fixtures ───────────────────────────────────────────────────────────── + +const over = (n: number, c = 'x') => c.repeat(n + 1); +const PAGE_FACTS_FIXTURES: Array<[string, string]> = [ + [ + 'facts: head facts', + doc( + ` + Blue Widget | Example Store + + + + `, + `

+ Blue + Widget \t2000 +

second h1 is ignored

` + ), + ], + ['facts: empty document', doc('')], + [ + 'facts: empty values', + doc(' ', '

\n

'), + ], + [ + 'facts: Product with brand object, image array, rating, per-SKU offers', + doc( + ld({ + '@context': 'https://schema.org', + '@type': 'Product', + 'name': 'Blue Widget', + 'brand': { '@type': 'Brand', 'name': 'Example Brand' }, + 'image': ['https://www.example.com/img/1.jpg', 'https://www.example.com/img/2.jpg'], + 'aggregateRating': { '@type': 'AggregateRating', 'ratingValue': 4.6, 'ratingCount': 212, 'reviewCount': 90 }, + 'offers': [ + { sku: 'SKU-B', price: 35.99, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { sku: 1234567, price: '29.99', priceCurrency: 'USD', availability: 'http://schema.org/OutOfStock/' }, + { sku: 'SKU-A', price: 0, priceCurrency: 'USD', availability: 'PreOrder' }, + ], + }) + ), + ], + [ + 'facts: ProductGroup inside @graph with an AggregateOffer', + doc( + ld({ + '@context': 'https://schema.org', + '@graph': [ + { '@type': 'WebPage', 'name': 'not the product' }, + { + '@type': 'ProductGroup', + 'name': 'Widget Family', + 'brand': 'Example Brand', + 'image': { '@type': 'ImageObject', 'url': 'https://www.example.com/img/family.jpg' }, + 'aggregateRating': { ratingValue: '4.2', reviewCount: '37' }, + 'offers': { + '@type': 'AggregateOffer', + 'lowPrice': 10, + 'highPrice': 20, + 'offers': [ + { sku: 'S', price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { sku: 'L', price: 20, priceCurrency: 'USD', availability: 'https://schema.org/LimitedAvailability' }, + ], + }, + }, + ], + }) + ), + ], + [ + 'facts: top-level array, @type array, first product wins', + doc( + ld([ + { '@type': 'WebSite', 'name': 'Example Store' }, + { '@type': ['Product', 'Thing'], 'name': 'First', 'image': 'https://www.example.com/first.jpg' }, + ]) + ld({ '@type': 'Product', 'name': 'Second' }) + ), + ], + [ + 'facts: malformed, null and string blocks beside valid ones', + doc( + rawLd('{ "@type": "Product", "name": broken ') + + ld({ '@type': 'Product', 'name': 'Survivor', 'offers': { sku: 'X', price: 5, availability: 'InStock' } }) + + rawLd('null') + + rawLd('"just a string"') + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }) + ), + ], + [ + 'facts: a Product that states nothing else', + doc( + ld({ + '@type': 'Product', + 'brand': { '@type': 'Brand' }, + 'image': [], + 'aggregateRating': { ratingValue: 'n/a', ratingCount: null }, + 'offers': { '@type': 'AggregateOffer', 'lowPrice': 1, 'highPrice': 9 }, + }) + ), + ], + ['facts: rating value only', doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: 5 } }))], + [ + 'facts: rating null count falls back to reviewCount', + doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: '3.5', ratingCount: null, reviewCount: 8 } })), + ], + ['facts: rating zero', doc(ld({ '@type': 'Product', 'aggregateRating': { ratingValue: 0, ratingCount: 0 } }))], + [ + 'facts: odd offer field types', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { price: 12 }, + { sku: { nested: true }, price: true, priceCurrency: 840, availability: 42 }, + 'not an offer', + null, + ], + }) + ), + ], + [ + 'facts: breadcrumb ordering', + doc( + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': [ + { '@type': 'ListItem', 'position': 3, 'item': { '@id': 'https://www.example.com/c/w', 'name': 'Widgets' } }, + { '@type': 'ListItem', 'position': '1', 'name': 'Home', 'item': 'https://www.example.com/' }, + { '@type': 'ListItem', 'name': 'Unpositioned' }, + { '@type': 'ListItem', 'position': 2, 'name': 'Shop', 'item': { name: '' } }, + { '@type': 'ListItem', 'position': 4 }, + { '@type': 'ListItem', 'position': 5, 'name': 'Element name', 'item': { name: 'Item name' } }, + ], + }) + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'second list is ignored' }] }) + ), + ], + [ + 'facts: breadcrumbs with no named crumbs', + doc(ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1 }] })), + ], + [ + 'facts: strings past and at maxString', + doc( + `${over(PAGE_FACT_BOUNDS.maxString, 'b')} + ` + + ld({ + '@type': 'Product', + 'name': over(PAGE_FACT_BOUNDS.maxString, 'b'), + 'brand': 'a'.repeat(PAGE_FACT_BOUNDS.maxString), + 'image': over(PAGE_FACT_BOUNDS.maxString, 'b'), + }), + `

${over(PAGE_FACT_BOUNDS.maxString, 'b')}

` + ), + ], + [ + 'facts: exactly maxOffers offers', + doc(ld({ '@type': 'Product', 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => foffer(i)) })), + ], + [ + 'facts: maxOffers + 1 across an AggregateOffer', + doc( + ld({ + '@type': 'Product', + 'name': 'Kept', + 'offers': [ + foffer(-1), + { + '@type': 'AggregateOffer', + 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => foffer(i)), + }, + ], + }) + ), + ], + ['facts: exactly maxBreadcrumbs', doc(crumbs(PAGE_FACT_BOUNDS.maxBreadcrumbs))], + ['facts: maxBreadcrumbs + 1', doc(crumbs(PAGE_FACT_BOUNDS.maxBreadcrumbs + 1))], + [ + 'facts: one overlong breadcrumb name', + doc( + ld({ + '@type': 'BreadcrumbList', + 'itemListElement': [ + { position: 1, name: 'Home' }, + { position: 2, name: over(PAGE_FACT_BOUNDS.maxString) }, + ], + }) + ), + ], + [ + 'facts: offer fields past and at maxOfferField', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { + sku: over(PAGE_FACT_BOUNDS.maxOfferField), + price: over(PAGE_FACT_BOUNDS.maxOfferField), + priceCurrency: over(PAGE_FACT_BOUNDS.maxOfferField), + availability: `https://schema.org/${over(PAGE_FACT_BOUNDS.maxOfferField)}`, + }, + { + sku: 'k'.repeat(PAGE_FACT_BOUNDS.maxOfferField), + price: '1.00', + priceCurrency: 'USD', + availability: 'https://schema.org/InStock', + }, + ], + }) + ), + ], + [ + 'facts: one sku-less offer takes the product sku', + doc( + ld({ + '@type': 'Product', + 'sku': 9001, + 'offers': { price: 5, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + }) + ), + ], + [ + 'facts: one aggregated offer with an empty sku', + doc( + ld({ + '@type': 'Product', + 'sku': 'P-1', + 'offers': { '@type': 'AggregateOffer', 'offers': [{ price: 5, priceCurrency: 'USD', sku: '' }] }, + }) + ), + ], + ['facts: the offer sku wins', doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': { sku: 'O-1', price: 5 } }))], + [ + 'facts: several offers never take the product sku', + doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': [{ price: 5 }, { sku: 'O-2', price: 6 }] })), + ], + [ + 'facts: a refused offer sku is not replaced', + doc(ld({ '@type': 'Product', 'sku': 'P-1', 'offers': { sku: over(PAGE_FACT_BOUNDS.maxOfferField), price: 5 } })), + ], + [ + 'facts: an overlong inherited sku', + doc(ld({ '@type': 'Product', 'sku': over(PAGE_FACT_BOUNDS.maxOfferField, 'y'), 'offers': { price: 5 } })), + ], + [ + 'facts: ProductGroup reads its hasVariant offers', + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Widget Family', + 'productGroupID': 'FAM-1', + 'hasVariant': [ + variant( + 'V-S', + { price: 10, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + { + name: 'Widget, small', + brand: { '@type': 'Brand', 'name': 'Variant Brand' }, + image: 'https://www.example.com/img/small.jpg', + aggregateRating: { ratingValue: 4, ratingCount: 3 }, + } + ), + variant('V-L', [ + { price: 20, priceCurrency: 'USD', availability: 'https://schema.org/OutOfStock' }, + { sku: 'V-L-REFURB', price: 15, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + ]), + 'not a variant', + ], + }) + ), + ], + [ + "facts: a ProductGroup's own offers and fields win", + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Group name', + 'brand': 'Group Brand', + 'image': 'https://www.example.com/img/group.jpg', + 'aggregateRating': { ratingValue: 5, reviewCount: 1 }, + 'offers': { '@type': 'AggregateOffer', 'offers': [{ sku: 'G-1', price: 1 }] }, + 'hasVariant': [variant('V-1', { price: 2 }, { name: 'Variant name', brand: 'Variant Brand' })], + }) + ), + ], + [ + 'facts: a ProductGroup value past its bound', + doc( + ld({ + '@type': 'ProductGroup', + 'name': over(PAGE_FACT_BOUNDS.maxString, 'n'), + 'hasVariant': [variant('V-1', { price: 2 }, { name: 'Variant name' })], + }) + ), + ], + [ + 'facts: variant offers over the cap together', + doc( + ld({ + '@type': 'ProductGroup', + 'hasVariant': [ + variant( + 'A', + Array.from({ length: PAGE_FACT_BOUNDS.maxOffers }, (_, i) => ({ price: i })) + ), + variant('B', [{ price: 0 }]), + ], + }) + ), + ], + [ + 'facts: a refused group never falls through', + doc( + ld({ + '@type': 'ProductGroup', + 'offers': Array.from({ length: PAGE_FACT_BOUNDS.maxOffers + 1 }, (_, i) => ({ price: i })), + 'hasVariant': [variant('A', { price: 1 })], + }) + ), + ], + [ + 'facts: mainEntity object', + doc( + ld({ + '@type': 'WebPage', + 'mainEntity': { '@type': 'Product', 'name': 'Nested', 'offers': { sku: 'N-1', price: 3 } }, + }) + ), + ], + [ + 'facts: mainEntity array inside @graph', + doc( + ld({ + '@context': 'https://schema.org', + '@graph': [ + { + '@type': 'ItemPage', + 'mainEntity': [ + { '@type': 'Organization', 'name': 'not it' }, + { '@type': 'ProductGroup', 'name': 'Nested group' }, + ], + }, + ], + }) + ), + ], + [ + 'facts: mainEntityOfPage', + doc(ld({ '@type': 'WebPage', 'mainEntityOfPage': { '@type': 'Product', 'name': 'Of page' } })), + ], + [ + 'facts: nested breadcrumb', + doc( + ld({ + '@type': 'CollectionPage', + 'breadcrumb': { + '@type': 'BreadcrumbList', + 'itemListElement': [ + { position: 2, name: 'Widgets' }, + { position: 1, name: 'Home' }, + ], + }, + }) + ), + ], + [ + 'facts: top-level beats nested', + doc( + ld({ + '@type': 'WebPage', + 'mainEntity': { '@type': 'Product', 'name': 'Nested' }, + 'breadcrumb': { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Nested crumb' }] }, + }) + + ld({ '@type': 'Product', 'name': 'Top level' }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Top crumb' }] }) + ), + ], + [ + 'facts: one level deep, only under page nodes', + doc( + ld({ + '@graph': [ + { + '@type': 'WebPage', + 'mainEntity': { '@type': 'WebPage', 'mainEntity': { '@type': 'Product', 'name': 'Deep' } }, + }, + { + '@type': 'Organization', + 'mainEntity': { '@type': 'Product', 'name': 'Under an organization' }, + 'breadcrumb': { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'no' }] }, + }, + { + '@type': 'WebPage', + 'mainEntityOfPage': 'https://www.example.com/p', + 'breadcrumb': { itemListElement: [{ position: 1, name: 'untyped' }] }, + }, + ], + }) + ), + ], +]; + +// ── new edge cases: where the two readers' walks DIVERGE, so the shared parse must serve both ── + +const EDGE_FIXTURES: Array<[string, string]> = [ + [ + // pageFacts stops after block 0 (product + breadcrumbs found); structuredOffers must still read the + // later blocks — parsed only for it. + 'edge: facts stop early, offers read on', + doc( + ld({ + '@graph': [ + { '@type': 'Product', 'name': 'First', 'offers': { sku: 'A', price: 1, priceCurrency: 'USD' } }, + { '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }, + ], + }) + + rawLd('{ broken') + + ld({ '@type': 'Product', 'name': 'Second', 'offers': [{ price: 2 }, { price: 3, availability: 'InStock' }] }) + + ld([{ '@type': 'Product', 'offers': { price: '0.50', priceCurrency: 'EUR' } }]) + ), + ], + [ + // structuredOffers overflows in block 1; pageFacts still needs block 3 for its breadcrumbs. + 'edge: offers overflow early, facts read on', + doc( + ld({ '@type': 'Organization', 'name': 'Example' }) + + ld({ + '@graph': Array.from({ length: STRUCTURED_OFFER_CAP + 1 }, (_, i) => ({ + '@type': 'Product', + 'offers': soffer(i), + })), + }) + + ld({ '@type': 'WebSite' }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Late crumb' }] }) + ), + ], + [ + 'edge: exactly the offer cap across two products', + doc( + ld({ + '@graph': [ + { '@type': 'Product', 'offers': Array.from({ length: 150 }, (_, i) => soffer(i)) }, + { + '@type': 'Product', + 'offers': Array.from({ length: STRUCTURED_OFFER_CAP - 150 }, (_, i) => soffer(i + 150)), + }, + ], + }) + ), + ], + [ + 'edge: one offer past the cap across two blocks', + doc( + ld({ '@type': 'Product', 'offers': Array.from({ length: STRUCTURED_OFFER_CAP }, (_, i) => soffer(i)) }) + + ld({ '@type': 'Product', 'offers': soffer(999) }) + ), + ], + [ + // A ProductGroup is not a `Product` to structuredOffers (null), while pageFacts reads its variants. + 'edge: ProductGroup — offers see nothing, facts see the variants', + doc( + ld({ + '@type': 'ProductGroup', + 'name': 'Family', + 'hasVariant': [variant('V1', { price: 4, priceCurrency: 'USD', availability: 'https://schema.org/InStock' })], + }) + ), + ], + [ + 'edge: JSON primitives, empty and whitespace blocks', + doc( + rawLd('') + + rawLd(' \n ') + + rawLd('42') + + rawLd('true') + + rawLd('[]') + + rawLd('[null, 1, "x", []]') + + rawLd('{"@graph": {"@type": "Product", "offers": {"price": 1}}}') + + ld({ '@type': 'Product', 'name': 'After the junk', 'offers': { price: 8 } }) + ), + ], + [ + 'edge: over-long strings everywhere', + doc( + `${'t'.repeat(5000)}` + + ld({ + '@type': 'Product', + 'name': 'n'.repeat(3000), + 'sku': 's'.repeat(65), + 'offers': [ + { + price: 'p'.repeat(100000), + priceCurrency: 'c'.repeat(70), + availability: `https://schema.org/${'a'.repeat(80)}`, + }, + { price: 1, priceCurrency: 'USD', availability: 'https://schema.org/InStock' }, + ], + }), + `

${'h '.repeat(1500)}

` + ), + ], + [ + 'edge: sort order with nulls, duplicates and non-ASCII', + doc( + ld({ + '@type': 'Product', + 'offers': [ + { price: 'Ü', priceCurrency: 'USD' }, + { price: 'Z' }, + { availability: 'InStock' }, + { price: 'Z' }, + { price: '10', priceCurrency: 'USD', availability: 'https://schema.org/OutOfStock' }, + { price: '9', priceCurrency: 'usd', availability: '/' }, + { price: 1e21, priceCurrency: '€', availability: 'https://schema.org/' }, + { price: -0, priceCurrency: 'USD', availability: '' }, + ], + 'aggregateRating': { ratingValue: -0, ratingCount: ' 12 ' }, + }) + ), + ], + [ + 'edge: many small blocks', + doc( + Array.from({ length: 60 }, (_, i) => + i % 7 === 3 + ? rawLd('{ nope') + : ld({ '@type': i % 5 ? 'Thing' : 'Product', 'name': `N${i}`, 'offers': { price: i } }) + ).join('') + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Last' }] }) + ), + ], +]; + +// ── pages that break one reader: each side must null exactly what it nulled before ───────────── + +const hostile = (script: string) => + doc( + `Hostile` + + ld({ '@type': 'Product', 'name': 'H', 'offers': { sku: 'H1', price: 2, priceCurrency: 'USD' } }) + + ld({ '@type': 'BreadcrumbList', 'itemListElement': [{ position: 1, name: 'Home' }] }), + `

Hostile

` + ); +const HOSTILE_FIXTURES: Array<[string, string, (c: Claims) => void]> = [ + [ + 'hostile: document.title throws — facts null, offers kept', + hostile(`Object.defineProperty(document, 'title', { get() { throw new Error('hostile'); } });`), + (c) => { + assert.equal(c.pageFacts, null); + assert.deepEqual(c.structuredOffers, ['2', 'USD', null]); + }, + ], + [ + 'hostile: document.querySelector throws — facts null, offers kept', + hostile(`document.querySelector = () => { throw new Error('hostile'); };`), + (c) => { + assert.equal(c.pageFacts, null); + assert.deepEqual(c.structuredOffers, ['2', 'USD', null]); + }, + ], + [ + 'hostile: Array.prototype.flat throws — offers null, facts kept', + hostile(`Array.prototype.flat = function () { throw new Error('hostile'); };`), + (c) => { + assert.equal(c.structuredOffers, null); + assert.equal(c.pageFacts?.product?.name, 'H'); + assert.deepEqual(c.pageFacts?.breadcrumbs, ['Home']); + }, + ], + [ + 'hostile: document.querySelectorAll throws — both null', + hostile(`document.querySelectorAll = () => { throw new Error('hostile'); };`), + (c) => assert.deepEqual(c, { structuredOffers: null, pageFacts: null }), + ], + [ + 'hostile: one block’s textContent throws — that block is skipped by both', + hostile( + `Object.defineProperty(document.querySelector('script[type="application/ld+json"]'), 'textContent', { get() { throw new Error('hostile'); } });` + ), + (c) => { + assert.equal(c.structuredOffers, null, 'the only Product block was unreadable'); + assert.equal(c.pageFacts?.product, null); + assert.deepEqual(c.pageFacts?.breadcrumbs, ['Home']); + }, + ], +]; + +test('every structuredOffers-suite fixture: merged output is identical to the 1.37.0 pair', async () => { + for (const [label, html] of STRUCTURED_OFFERS_FIXTURES) await compare(label, html); +}); + +test('every pageFacts-suite fixture: merged output is identical to the 1.37.0 pair', async () => { + for (const [label, html] of PAGE_FACTS_FIXTURES) await compare(label, html); +}); + +test('edge cases where the two walks diverge: merged output is identical to the 1.37.0 pair', async () => { + const results = new Map(); + for (const [label, html] of EDGE_FIXTURES) results.set(label, await compare(label, html)); + // Not vacuous: the divergent walks really did produce both claims. + const early = results.get('edge: facts stop early, offers read on')!; + assert.equal(early.structuredOffers?.length, 4 * 3, 'offers from blocks past the one facts stopped at'); + assert.deepEqual(early.pageFacts?.breadcrumbs, ['Home']); + const late = results.get('edge: offers overflow early, facts read on')!; + assert.equal(late.structuredOffers, null); + assert.deepEqual(late.pageFacts?.breadcrumbs, ['Late crumb']); +}); + +test('a page that breaks one reader: each claim is null exactly where the 1.37.0 pair made it null', async () => { + for (const [label, html, check] of HOSTILE_FIXTURES) check(await compare(label, html)); +}); + +// ── seeded fuzz ──────────────────────────────────────────────────────────────────────────────── + +/** mulberry32 — a fixed seed, so a failure reproduces. */ +const prng = (seed: number) => () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +}; + +const fuzzDocuments = (count: number, seed: number): string[] => { + const rand = prng(seed); + const pick = (xs: readonly T[]): T => xs[Math.floor(rand() * xs.length)]; + const maybe = (p: number) => rand() < p; + const STRINGS = [ + 'Widget', + '', + ' padded ', + 'Ünïcødé ✓', + 'x'.repeat(64), + 'x'.repeat(65), + 'y'.repeat(PAGE_FACT_BOUNDS.maxString), + 'z'.repeat(PAGE_FACT_BOUNDS.maxString + 1), + 'a/b/', + '35.99', + ' 4.5 ', + 'n/a', + ]; + const SCALARS = [0, -0, 12.5, 1e21, -3, 1234567, true, false, null, {}, [], { name: 'obj' }]; + const TYPES = [ + 'Product', + 'Product', + 'ProductGroup', + 'BreadcrumbList', + 'WebPage', + 'ItemPage', + 'CollectionPage', + 'Organization', + 'Offer', + 'AggregateOffer', + 'Thing', + ['Product', 'Thing'], + ['ProductGroup'], + ['WebPage', 'ItemPage'], + undefined, + 42, + ]; + const AVAILABILITY = [ + 'https://schema.org/InStock', + 'http://schema.org/OutOfStock/', + 'InStock', + 'PreOrder', + '', + '/', + 'https://schema.org/', + 42, + null, + 'x'.repeat(70), + `https://schema.org/${'x'.repeat(70)}`, + ]; + const value = () => (maybe(0.6) ? pick(STRINGS) : pick(SCALARS)); + const offer = (nested = false): unknown => { + if (maybe(0.05)) return pick(['not an offer', null, 7]); + const o: Record = {}; + if (maybe(0.3)) o['@type'] = pick(['Offer', 'AggregateOffer']); + if (maybe(0.85)) o.price = maybe(0.6) ? pick([9.99, 0, -0, '35.99', 1e21, 100, '1,299.00']) : value(); + if (maybe(0.7)) o.priceCurrency = maybe(0.8) ? pick(['USD', 'EUR', 'usd', '€']) : value(); + if (maybe(0.7)) o.availability = pick(AVAILABILITY); + if (maybe(0.5)) o.sku = maybe(0.7) ? `SKU-${Math.floor(rand() * 50)}` : value(); + if (!nested && o['@type'] === 'AggregateOffer' && maybe(0.7)) o.offers = offers(true); + return o; + }; + const offerCount = () => pick([0, 1, 1, 1, 2, 2, 3, 5, 8, 199, 200, 201]); + const offers = (nested = false): unknown => { + const list = Array.from({ length: offerCount() }, () => offer(nested)); + switch (pick(['array', 'array', 'single', 'aggregate', 'mixed', 'absent'])) { + case 'single': + return list[0] ?? offer(nested); + case 'aggregate': + return { '@type': 'AggregateOffer', 'lowPrice': 1, 'highPrice': 9, 'offers': maybe(0.8) ? list : undefined }; + case 'mixed': + return [offer(nested), { '@type': 'AggregateOffer', 'offers': list }, ...list.slice(0, 2)]; + case 'absent': + return pick([undefined, null, '', 'string offers']); + default: + return list; + } + }; + const crumbList = (): unknown => { + const n = pick([0, 1, 2, 3, 5, 30, 31]); + return { + '@type': maybe(0.9) ? 'BreadcrumbList' : undefined, + 'itemListElement': Array.from({ length: n }, (_, i) => + maybe(0.05) + ? 'junk' + : { + '@type': 'ListItem', + 'position': maybe(0.8) ? pick([i + 1, String(i + 1), n - i, 1, null, 'x']) : undefined, + 'name': maybe(0.8) ? pick([`C${i}`, '', 'Home', 'z'.repeat(PAGE_FACT_BOUNDS.maxString + 1)]) : undefined, + 'item': maybe(0.4) ? pick([{ name: `Item ${i}` }, { name: '' }, 'https://www.example.com/']) : undefined, + } + ), + }; + }; + const node = (depth: number): unknown => { + if (maybe(0.04)) return pick([null, 'text', 3, []]); + const type = pick(TYPES); + const n: Record = { '@type': type }; + if (maybe(0.7)) n.name = value(); + if (maybe(0.4)) n.brand = maybe(0.5) ? { '@type': 'Brand', 'name': value() } : value(); + if (maybe(0.4)) + n.image = pick([ + value(), + [value(), 'second'], + { '@type': 'ImageObject', 'url': value() }, + [], + [{ url: 'https://www.example.com/i.jpg' }], + ]); + if (maybe(0.4)) + n.aggregateRating = { ratingValue: value(), ratingCount: maybe(0.5) ? value() : undefined, reviewCount: value() }; + if (maybe(0.5)) n.sku = maybe(0.6) ? `P-${Math.floor(rand() * 20)}` : value(); + if (maybe(0.7)) n.offers = offers(); + if (maybe(0.2) || type === 'ProductGroup') + n.hasVariant = Array.from({ length: pick([0, 1, 2, 3]) }, () => node(depth + 1)); + if (type === 'BreadcrumbList' || maybe(0.1)) + n.itemListElement = (crumbList() as Record).itemListElement; + if (depth < 2 && maybe(0.25)) n.mainEntity = maybe(0.5) ? node(depth + 1) : [node(depth + 1), node(depth + 1)]; + if (depth < 2 && maybe(0.15)) n.mainEntityOfPage = maybe(0.6) ? node(depth + 1) : 'https://www.example.com/p'; + if (depth < 2 && maybe(0.2)) n.breadcrumb = crumbList(); + return n; + }; + const block = (): string => { + switch (pick(['node', 'node', 'node', 'array', 'graph', 'graph', 'malformed', 'primitive', 'graph-object'])) { + case 'array': + return ld(Array.from({ length: pick([0, 1, 2, 4]) }, () => node(0))); + case 'graph': + return ld({ + '@context': 'https://schema.org', + '@graph': Array.from({ length: pick([1, 2, 3, 6]) }, () => node(0)), + }); + case 'graph-object': + return ld({ '@graph': node(0) }); + case 'malformed': + return rawLd(pick(['{ "@type": "Product", ', '', ' ', '{nope}', '[1,'])); + case 'primitive': + return rawLd(pick(['null', '"str"', '42', 'true', '[]'])); + default: + return ld(node(0)); + } + }; + const text = () => + pick(['Blue Widget', '', ' spaced out ', 'w'.repeat(PAGE_FACT_BOUNDS.maxString + 1), 'Ünïcødé']); + return Array.from({ length: count }, () => { + const head = [ + maybe(0.8) ? `${text()}` : '', + maybe(0.6) ? `` : '', + maybe(0.6) + ? `` + : '', + ...Array.from({ length: pick([0, 1, 1, 2, 3, 5]) }, block), + ].join(''); + const body = maybe(0.7) ? `

${text()} ${text()}

` : ''; + return `${head}${body}`; + }); +}; + +test('seeded fuzz: 400 generated documents, merged output identical to the 1.37.0 pair on every one', async () => { + const documents = fuzzDocuments(400, 0x5eed138); + const page = await browser.newPage(); + const seen = { offers: 0, offersNull: 0, product: 0, productOffers: 0, breadcrumbs: 0, capRefusal: 0 }; + try { + pages.set('/fuzz', doc('')); + await page.goto(`${base}/fuzz`, { waitUntil: 'load' }); + for (let i = 0; i < documents.length; i++) { + await page.evaluate((html) => { + document.documentElement.innerHTML = html; + }, documents[i]); + const old = await before137(page); + const merged = await after138(page); + assertParity(old, merged, `fuzz #${i}`); + if (merged.structuredOffers) seen.offers++; + else seen.offersNull++; + if (merged.pageFacts?.product) seen.product++; + if (merged.pageFacts?.product?.offers) seen.productOffers++; + if (merged.pageFacts?.breadcrumbs) seen.breadcrumbs++; + if (documents[i].split('"price"').length > STRUCTURED_OFFER_CAP && !merged.structuredOffers) seen.capRefusal++; + } + } finally { + await page.close(); + } + // The corpus must actually exercise both readers, not compare a wall of nulls. + for (const [what, n] of Object.entries(seen)) assert.ok(n >= 20, `fuzz coverage: only ${n} documents with ${what}`); +}); + +// ── the 1.37.0 extractors, VERBATIM (renamed only) ───────────────────────────────────────────── +// `extractStructuredOffers` from src/renderer.ts and `extractPageFacts` from src/pageFacts.ts at +// browser 1.37.0. Do not edit: they are the reference the merged extractor is held to. + +function legacyStructuredOffers(cap: number): Array | null { + const triples: Array> = []; + let overflowed = false; + // Field-level byte bound: the cap bounds triple COUNT, so without this a single pathological + // field (a megabyte "price" string) would still inflate every posted result for the page. No + // legitimate price, currency code, or availability token approaches 64 characters. + const bound = (value: string): string | null => (value === '' ? null : value.slice(0, 64)); + const collect = (node: unknown) => { + if (overflowed || !node || typeof node !== 'object') return; + const record = node as Record; + const type = record['@type']; + const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product')); + if (!isProduct) return; + const raw = record.offers; + const list = Array.isArray(raw) ? raw : raw ? [raw] : []; + for (const entry of list) { + if (!entry || typeof entry !== 'object') continue; + if (triples.length >= cap) { + overflowed = true; + return; + } + const offer = entry as Record; + // filter(Boolean) before pop: a trailing slash (https://schema.org/InStock/) would + // otherwise pop the empty segment and read as no availability at all. + const availability = + typeof offer.availability === 'string' + ? bound(offer.availability.split('/').filter(Boolean).pop() ?? '') + : null; + const price = offer.price === undefined || offer.price === null ? null : bound(String(offer.price)); + const currency = typeof offer.priceCurrency === 'string' ? bound(offer.priceCurrency) : null; + triples.push([price, currency, availability]); + } + }; + document.querySelectorAll('script[type="application/ld+json"]').forEach((el) => { + if (overflowed) return; + let data: unknown; + try { + data = JSON.parse(el.textContent || ''); + } catch { + return; // one malformed block must not cost the page its other blocks + } + const graph = (data as Record)?.['@graph']; + const nodes = Array.isArray(data) ? data : Array.isArray(graph) ? graph : [data]; + for (const node of nodes) { + if (overflowed) break; + collect(node); + } + }); + if (overflowed || !triples.length) return null; + // Field-wise, not JSON.stringify per comparison: sorting is O(n log n) COMPARISONS, so + // stringifying inside the comparator serialises every triple many times over. Code-unit + // comparison, not localeCompare: the whole point of the sort is a sequence that is identical + // across renders, and collation varies with the browser's locale/ICU. + triples.sort((a, b) => { + for (let i = 0; i < 3; i++) { + const x = a[i]; + const y = b[i]; + if (x === y) continue; + if (x === null) return -1; + if (y === null) return 1; + return x < y ? -1 : 1; + } + return 0; + }); + return triples.flat(); +} + +function legacyPageFacts(bounds: PageFactBounds): PageFacts { + type Json = Record; + const isObject = (value: unknown): value is Json => !!value && typeof value === 'object' && !Array.isArray(value); + const boundTo = + (max: number) => + (value: unknown): string | null => + typeof value !== 'string' || value === '' || value.length > max ? null : value; + const str = boundTo(bounds.maxString); + const offerField = boundTo(bounds.maxOfferField); + const hasType = (node: Json, wanted: string[]) => { + const type = node['@type']; + return Array.isArray(type) ? type.some((t) => wanted.includes(t as string)) : wanted.includes(type as string); + }; + // A schema.org number may be written as a JSON number or a numeric string ("4.5"). Anything else — + // including null, which Number() would silently read as 0 — is no claim. + const num = (value: unknown): number | null => { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!trimmed || trimmed.length > bounds.maxOfferField) return null; + const n = Number(trimmed); + return Number.isFinite(n) ? n : null; + }; + const listOf = (value: unknown): unknown[] => (Array.isArray(value) ? value : value ? [value] : []); + + const canonicalEl = document.querySelector('link[rel="canonical"]') as HTMLLinkElement | null; + const descriptionEl = document.querySelector('meta[name="description" i]'); + const h1El = document.querySelector('h1'); + + // schema.org WebPage and its subtypes: the nodes whose `mainEntity` / `breadcrumb` are searched. + const PAGE_TYPES = [ + 'WebPage', + 'ItemPage', + 'CollectionPage', + 'SearchResultsPage', + 'ProfilePage', + 'AboutPage', + 'ContactPage', + 'FAQPage', + 'QAPage', + 'CheckoutPage', + 'MedicalWebPage', + 'RealEstateListing', + 'MediaGallery', + 'ImageGallery', + 'VideoGallery', + ]; + const PRODUCT_TYPES = ['Product', 'ProductGroup']; + + // A node at the top of a block (or of its @graph) always wins; one found a single level down — a + // page node's `mainEntity` / `mainEntityOfPage` / `breadcrumb` — is used only when no top-level node + // matched. One level only: this reads what a page states about itself, it does not crawl a graph. + let productNode: Json | null = null; + let breadcrumbNode: Json | null = null; + let nestedProduct: Json | null = null; + let nestedBreadcrumb: Json | null = null; + const blocks = document.querySelectorAll('script[type="application/ld+json"]'); + for (let i = 0; i < blocks.length && !(productNode && breadcrumbNode); i++) { + let data: unknown; + try { + data = JSON.parse(blocks[i].textContent || ''); + } catch { + continue; // one malformed block must not cost the page its other blocks + } + const graph = isObject(data) ? data['@graph'] : undefined; + const nodes = Array.isArray(data) ? data : Array.isArray(graph) ? graph : [data]; + for (const node of nodes) { + if (!isObject(node)) continue; + if (!productNode && hasType(node, PRODUCT_TYPES)) productNode = node; + if (!breadcrumbNode && hasType(node, ['BreadcrumbList'])) breadcrumbNode = node; + if (!hasType(node, PAGE_TYPES)) continue; + if (!nestedProduct) { + for (const entity of listOf(node.mainEntity)) { + if (isObject(entity) && hasType(entity, PRODUCT_TYPES)) { + nestedProduct = entity; + break; + } + } + } + const ofPage = node.mainEntityOfPage; + if (!nestedProduct && isObject(ofPage) && hasType(ofPage, PRODUCT_TYPES)) nestedProduct = ofPage; + const crumb = node.breadcrumb; + if (!nestedBreadcrumb && isObject(crumb) && hasType(crumb, ['BreadcrumbList'])) nestedBreadcrumb = crumb; + } + } + productNode = productNode ?? nestedProduct; + breadcrumbNode = breadcrumbNode ?? nestedBreadcrumb; + + // A JSON number is stated as its string form ("35.99", "12345"), as structuredOffers states a price. + const asString = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? String(value) : value); + const absent = (value: unknown) => value === undefined || value === null || value === ''; + const nonEmpty = (value: unknown) => (typeof value === 'string' && value !== '' ? value : undefined); + + // The offer objects a node states, in document order — an AggregateOffer summarises + // (lowPrice/highPrice), so only the offers it lists are offers. null when past the bound. + const offersOf = (raw: unknown): Json[] | null => { + const found: Json[] = []; + for (const entry of listOf(raw)) { + const list = isObject(entry) && hasType(entry, ['AggregateOffer']) ? listOf(entry.offers) : [entry]; + for (const offer of list) { + if (!isObject(offer)) continue; + if (found.length >= bounds.maxOffers) return null; + found.push(offer); + } + } + return found; + }; + const tuple = (offer: Json, sku: unknown): PageFactOffer => { + // Reduced exactly as extractStructuredOffers reduces it: the last non-empty path segment, so a + // trailing slash (https://schema.org/InStock/) still yields the verdict. + const availability = + typeof offer.availability === 'string' ? (offer.availability.split('/').filter(Boolean).pop() ?? '') : null; + return [ + offerField(asString(sku)), + offerField(asString(offer.price)), + offerField(offer.priceCurrency), + offerField(availability), + ]; + }; + + const readOffers = (node: Json, variants: Json[]): PageFactOffer[] | null => { + const own = offersOf(node.offers); + if (own === null) return null; // refused — never fall through to the variants + if (own.length) { + // A product selling ONE offer often names the SKU on itself rather than on the offer. Only + // then: with several offers, the product's SKU names none of them. + const inherit = own.length === 1 && absent(own[0].sku); + return own.map((offer) => tuple(offer, inherit ? node.sku : offer.sku)); + } + // A ProductGroup usually sells through its variants (`hasVariant`, each a Product with its own + // offers and SKU) — read those when the group itself states no offers. + const out: PageFactOffer[] = []; + for (const variant of variants) { + const offers = offersOf(variant.offers); + if (offers === null) return null; + for (const offer of offers) { + if (out.length >= bounds.maxOffers) return null; + out.push(tuple(offer, absent(offer.sku) ? variant.sku : offer.sku)); + } + } + return out.length ? out : null; + }; + + // Each field is PICKED raw — from the node, else (for a ProductGroup) its first variant — and only + // then bounded, so a value the group states too long is refused rather than swapped for a variant's. + const pickBrand = (n: Json) => nonEmpty(isObject(n.brand) ? n.brand.name : n.brand); + const pickImage = (n: Json) => { + const image = Array.isArray(n.image) ? n.image[0] : n.image; + return nonEmpty(isObject(image) ? image.url : image); + }; + const pickRating = (n: Json): [number | null, number | null] | undefined => { + const r = n.aggregateRating; + if (!isObject(r)) return undefined; + const pair: [number | null, number | null] = [num(r.ratingValue), num(r.ratingCount ?? r.reviewCount)]; + return pair[0] !== null || pair[1] !== null ? pair : undefined; + }; + + const readProduct = (node: Json): PageFactsProduct => { + const variants = hasType(node, ['ProductGroup']) ? listOf(node.hasVariant).filter(isObject) : []; + const pick = (from: (n: Json) => T | undefined): T | undefined => + from(node) ?? (variants.length ? from(variants[0]) : undefined); + return { + name: str(pick((n) => nonEmpty(n.name))), + brand: str(pick(pickBrand)), + image: str(pick(pickImage)), + rating: pick(pickRating) ?? null, + offers: readOffers(node, variants), + }; + }; + + const readBreadcrumbs = (node: Json): string[] | null => { + const crumbs: Array<{ name: string; position: number | null; order: number }> = []; + const elements = listOf(node.itemListElement); + for (let order = 0; order < elements.length; order++) { + const el = elements[order]; + if (!isObject(el)) continue; + const itemName = isObject(el.item) ? el.item.name : undefined; + const raw = typeof itemName === 'string' && itemName !== '' ? itemName : el.name; + if (typeof raw !== 'string' || raw === '') continue; // an unnamed crumb claims nothing + // A name past the bound refuses the whole trail: a trail with a hole in it would disagree. + if (raw.length > bounds.maxString) return null; + if (crumbs.length >= bounds.maxBreadcrumbs) return null; + crumbs.push({ name: raw, position: num(el.position), order }); + } + // Ordered by position; a crumb without one keeps its document order after the positioned ones. + crumbs.sort((a, b) => + a.position === b.position + ? a.order - b.order + : a.position === null + ? 1 + : b.position === null + ? -1 + : a.position - b.position + ); + return crumbs.length ? crumbs.map((c) => c.name) : null; + }; + + return { + canonical: canonicalEl ? str(canonicalEl.href) : null, + title: str((document.title || '').trim()), + metaDescription: descriptionEl ? str(descriptionEl.getAttribute('content')) : null, + h1: h1El ? str((h1El.textContent || '').replace(/\s+/g, ' ').trim()) : null, + product: productNode ? readProduct(productNode) : null, + breadcrumbs: breadcrumbNode ? readBreadcrumbs(breadcrumbNode) : null, + }; +} diff --git a/packages/browser/test/pageFacts.test.ts b/packages/browser/test/pageFacts.test.ts index df48eced..9b67a99b 100644 --- a/packages/browser/test/pageFacts.test.ts +++ b/packages/browser/test/pageFacts.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; import puppeteer, { type Browser } from 'puppeteer'; -import { extractPageFacts, PAGE_FACT_BOUNDS } from '../dist/pageFacts.js'; +import { extractPageClaims, PAGE_FACT_BOUNDS, STRUCTURED_OFFER_CAP } from '../dist/pageFacts.js'; import { renderOnce } from '../dist/renderOnce.js'; /** @@ -50,7 +50,7 @@ const factsOf = async (html: string) => { const page = await browser.newPage(); try { await page.goto(`${base}${path}`, { waitUntil: 'load' }); - return await page.evaluate(extractPageFacts, PAGE_FACT_BOUNDS); + return (await page.evaluate(extractPageClaims, STRUCTURED_OFFER_CAP, PAGE_FACT_BOUNDS)).pageFacts; } finally { await page.close(); } @@ -655,3 +655,43 @@ test('an extraction that throws posts pageFacts: null — present, never a faile assert.notEqual(result.job.pageFacts, undefined); assert.deepEqual(result.job.structuredOffers, ['9.5', 'USD', 'InStock']); }); + +test('an offers reader that throws posts structuredOffers: null — pageFacts still read, never a failed render', async () => { + // Both claims come from ONE evaluate now; each reader is caught inside the page, so a throw in the + // offers reader (here: the page breaks Array.prototype.flat, which only that reader calls) must cost + // the consumer the offers and nothing else — what two separate evaluates gave. + pages.set( + '/hostile-offers', + RENDERED(`${base}/hostile-offers`).replace( + '', + `` + ) + ); + const result = await renderOnce({ url: `${base}/hostile-offers`, config: NO_SCROLL }); + assert.equal(result.outcome, 'ok'); + assert.ok(result.html && result.html.length > 0); + assert.equal(result.job.structuredOffers, null); + assert.notEqual(result.job.structuredOffers, undefined); + assert.equal(result.job.pageFacts?.product?.name, 'Blue Widget'); + assert.deepEqual(result.job.pageFacts?.product?.offers, [['W1', '9.5', 'USD', 'InStock']]); +}); + +test('a failure of the extraction evaluate itself posts null for BOTH claims — present, never a failed render', async () => { + // No in-page catch can see a result that cannot be returned: the page makes the offers reader's + // output cyclic, so the evaluate itself rejects. Both claims are then null (ran, failed benignly), + // never absent ("renderer predates it"), and the render still succeeds. + pages.set( + '/unreturnable', + RENDERED(`${base}/unreturnable`).replace( + '', + `` + ) + ); + const result = await renderOnce({ url: `${base}/unreturnable`, config: NO_SCROLL }); + assert.equal(result.outcome, 'ok'); + assert.ok(result.html && result.html.length > 0); + assert.equal(result.job.structuredOffers, null); + assert.notEqual(result.job.structuredOffers, undefined); + assert.equal(result.job.pageFacts, null); + assert.notEqual(result.job.pageFacts, undefined); +});