From 5571b240eedac47125fcbfffad7a0d650509885a Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:36:20 +0200 Subject: [PATCH] port main's MCP and privacy work to dev, before a merge can undo it Preparing the dev to main merge, the conflict triage turned up one file group where dev is not ahead but behind, and behind on a leak. `src/lib/mcp-tools.ts` and the MCP route exist on both branches and were written independently, so they conflict as add/add. main's copies carry five commits dev never received: a thousandfold error and a 35k-token catalogue (#2764), the search endpoint handing back an API key it had been asked to search for (#2766), two tools that answered nothing (#2765), a third annotation the submission scan reads (#2768), and a chain name outranking the subject of the question (#2770). Measured rather than inferred from the titles: `redact` appears 7 times in main's mcp-tools.ts and 3 times in its route, and 0 times in either of dev's. Resolving those conflicts in dev's favour, which a bulk resolution would do, reintroduces the leak. Checked before overwriting: the dev-to-main diff is additive. main adds category-aware scoring, a minimum score that cuts prefix-only noise, and redactSecrets(); the only lines it removes are the ones it replaces. dev holds nothing in these files that main lacks. The privacy page is the same shape without the security edge: main's is 219 lines against dev's 140 and is alone in documenting retention and the Vercel dependency. dev's predates it, from when the page existed only because the app directory requires a policy URL. Doing this now, in its own change, means the merge no longer has a resolution that can silently restore a credential leak. Gates: tsc clean, eslint clean, 567 tests (up from 561: main's test file carries six more). --- src/app/api/mcp/[transport]/route.ts | 109 +++++++++++-- src/app/privacy/page.tsx | 231 ++++++++++++++++++--------- src/lib/mcp-tools.test.ts | 44 ++++- src/lib/mcp-tools.ts | 46 +++++- 4 files changed, 340 insertions(+), 90 deletions(-) diff --git a/src/app/api/mcp/[transport]/route.ts b/src/app/api/mcp/[transport]/route.ts index 5ebe2826c..65850f548 100644 --- a/src/app/api/mcp/[transport]/route.ts +++ b/src/app/api/mcp/[transport]/route.ts @@ -22,6 +22,7 @@ import { compactRow, compareOnBenchmark, matchUseCase, + redactSecrets, searchBenchmarks, } from "@/lib/mcp-tools"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; @@ -67,6 +68,52 @@ export const runtime = "nodejs"; // we do it before letting the package parse the body. const MAX_BODY_BYTES = 64 * 1024; +/** + * Resolve a benchmark with only the filters it declares. + * + * A chain RPC bench IS a chain: `solana-rpc` declares no `chain` dimension, + * so `getBenchmark("solana-rpc", { chain: "solana" })` resolves to nothing + * and the caller sees unknown_slug. Passing a user's chain through blindly + * therefore empties the result for the most common question on the site. + * Drop a filter the bench does not declare instead of failing on it. + */ +async function resolveBenchmark( + slug: string, + want: { chain?: string; region?: string; tier?: string }, +): Promise<{ bench: Awaited>; applied: Record } > { + const aggregate = await getBenchmark(slug); + if (!aggregate) return { bench: undefined, applied: {} }; + const dims = aggregate.dimensions; + const applied: Record = {}; + + for (const key of ["chain", "region"] as const) { + const asked = want[key]; + if (!asked) continue; + const declared = dims?.[key]; + if (declared?.some((d) => d.value.toLowerCase() === asked.toLowerCase())) { + applied[key] = asked; + } + } + + // Tier resolves like the get_benchmark tool: the headline tier is the + // aggregate itself, so only a non-headline tier becomes a filter. + if (want.tier) { + const option = dims?.tier?.find((t) => t.value.toLowerCase() === want.tier?.toLowerCase()); + const headline = aggregate.aggregateFilters?.tier ?? dims?.tier?.[0]?.value; + if (option && option.value !== headline) applied.tier = option.value; + else if (option) applied.tier = option.value; + } + + const bench = await getBenchmark(slug, { + ...(applied.chain ? { chain: applied.chain } : {}), + ...(applied.region ? { region: applied.region } : {}), + ...(applied.tier && applied.tier !== (aggregate.aggregateFilters?.tier ?? dims?.tier?.[0]?.value) + ? { tier: applied.tier } + : {}), + }); + return { bench, applied }; +} + const mcpHandler = createMcpHandler( (server) => { server.registerTool( @@ -103,7 +150,7 @@ const mcpHandler = createMcpHandler( .optional() .describe("How many rows to return, 1 to 50. Default 25."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, async ({ category, limit }) => { const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); @@ -162,7 +209,7 @@ const mcpHandler = createMcpHandler( .describe("Optional category filter: RPCs, Trading, Bridges, Blockchains, Aggregators, RWA, NFT APIs."), limit: z.number().int().min(1).max(25).optional().describe("How many matches, 1 to 25. Default 8."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, async ({ query, category, limit }) => { const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); @@ -173,7 +220,9 @@ const mcpHandler = createMcpHandler( siteUrl: SITE.url, }); const payload = { - query, + // Never echo the caller's text back raw: asked to search for an API + // key, this endpoint returned the key inside its own payload. + query: redactSecrets(query), count: matches.length, matches, ...(matches.length === 0 @@ -243,7 +292,7 @@ const mcpHandler = createMcpHandler( .optional() .describe("Optional access cohort on chain RPC benchmarks: 'public' (default, free no-key endpoints) or 'keyed' (private, API-key providers). Only honored when the bench declares tier dimensions."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, async ({ slug, chain, region, tier }) => { const aggregate = await getBenchmark(slug); @@ -348,13 +397,18 @@ const mcpHandler = createMcpHandler( .min(2) .max(10) .describe("Two to ten provider names or slugs, as the user said them, e.g. ['Alchemy', 'QuickNode']."), - chain: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/).optional().describe("Optional chain slice when the bench declares chains."), + chain: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/).optional().describe("Optional chain slice when the bench declares chains. Ignored on a bench that is already one chain, like solana-rpc."), region: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/).optional().describe("Optional region slice when the bench declares regions."), + tier: z + .string() + .regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/) + .optional() + .describe("Access cohort on chain RPC benchmarks: 'public' (default, free no-key endpoints) or 'keyed' (API-key providers such as Alchemy, QuickNode, Chainstack, GetBlock). Named providers usually live on the keyed cohort."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, - async ({ benchmark, providers, chain, region }) => { - const b = await getBenchmark(benchmark, { chain, region }); + async ({ benchmark, providers, chain, region, tier }) => { + const { bench: b, applied } = await resolveBenchmark(benchmark, { chain, region, tier }); if (!b || b.editorialStatus !== "live") { const payload = { error: "unknown_slug", slug: benchmark }; return { @@ -364,12 +418,41 @@ const mcpHandler = createMcpHandler( }; } const comparison = compareOnBenchmark(b, providers, SITE.url); + + // A provider missing from this cohort is often measured on the other + // one: the default is the public endpoints, and the names people ask + // about (Alchemy, QuickNode) are keyed. Saying "not measured" when the + // answer is one tab away is the wrong answer, so look before saying it. + const otherTiers = (b.dimensions?.tier ?? []) + .map((t) => t.value) + .filter((v) => v !== (applied.tier ?? b.aggregateFilters?.tier ?? b.dimensions?.tier?.[0]?.value)); + const elsewhere: Record = {}; + if (comparison.missing.length > 0 && otherTiers.length > 0) { + for (const other of otherTiers) { + const alt = await getBenchmark(benchmark, { tier: other }); + if (!alt) continue; + for (const name of comparison.missing) { + if (elsewhere[name]) continue; + if (alt.results.some((r) => r.slug.toLowerCase() === name || r.name.toLowerCase() === name)) { + elsewhere[name] = other; + } + } + } + } + const payload = { ...comparison, title: b.title, + cohort: applied.tier ?? b.aggregateFilters?.tier ?? b.dimensions?.tier?.[0]?.value ?? null, headline: headlineSentence(b), quote: citationQuote(b, SITE.url), lowerIsBetter: !b.higherIsBetter, + ...(Object.keys(elsewhere).length > 0 + ? { + measuredOnAnotherCohort: elsewhere, + note: `Not on this cohort, but measured on: ${[...new Set(Object.values(elsewhere))].join(", ")}. Call again with that tier.`, + } + : {}), }; return { content: [{ type: "text", text: JSON.stringify(payload, null, 2) }], @@ -410,7 +493,7 @@ const mcpHandler = createMcpHandler( .describe("Chain they are building on, e.g. 'solana', 'base', 'arbitrum'. Narrows the benchmarks considerably."), region: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/).optional().describe("Optional region, e.g. 'eu-west', when latency from a location matters."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, async ({ use_case, chain, region }) => { const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); @@ -427,7 +510,9 @@ const mcpHandler = createMcpHandler( const detailed = await Promise.all( matches.slice(0, 3).map(async (m) => { - const full = await getBenchmark(m.slug, { chain, region }); + // Only the filters this bench declares: an RPC bench is already a + // chain, and passing one through would resolve to nothing. + const { bench: full } = await resolveBenchmark(m.slug, { chain, region }); if (!full || full.editorialStatus !== "live") return null; const top = isInsufficient(full) ? null : leader(full); return { @@ -445,7 +530,7 @@ const mcpHandler = createMcpHandler( ); const payload = { - useCase: use_case, + useCase: redactSecrets(use_case), ...(chain ? { chain } : {}), recognisedAs: uc?.id ?? null, guidance: @@ -493,7 +578,7 @@ const mcpHandler = createMcpHandler( .optional() .describe("Optional benchmark slug filter: return only the answers built on that bench."), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, async ({ benchmark }) => { const all = await loadRenderedAnswers(); diff --git a/src/app/privacy/page.tsx b/src/app/privacy/page.tsx index 89a10c29a..802a1ae6c 100644 --- a/src/app/privacy/page.tsx +++ b/src/app/privacy/page.tsx @@ -8,32 +8,36 @@ export const metadata: Metadata = pageMetadata({ path: "/privacy", title: "Privacy", description: - "What OpenChainBench collects: anonymous analytics, no accounts, no personal data, no advertising. Written from the code that runs the site.", + "What OpenChainBench collects, why, who processes it, how long it is kept, and what you can turn off. No accounts, no advertising, no data sold.", }); /** * Privacy policy. * - * Required to list the MCP server in the ChatGPT app directory (a Privacy - * Policy URL is a mandatory submission field), and it did not exist: both - * /privacy and /terms answered 404 on production. + * Required to list the MCP server in the ChatGPT app directory, and the + * review asks for five specific things: the categories of personal data + * collected, the purposes of use, the categories of recipients, retention + * timelines, and the controls offered to users. The sections below are + * ordered to match, because the first version was written as prose and the + * reviewer could not find them. * - * Every claim below is read off the code rather than drafted from a - * template, because a policy that overstates what we collect is as wrong as - * one that understates it: - * - `src/components/posthog-provider.tsx:57-64` sets `autocapture: false`, - * `capture_pageview: false` (pageviews are sent manually per route) and - * `person_profiles: "identified_only"`. Nothing in the codebase ever - * calls `posthog.identify`, so no person profile is ever created. - * - `api_host: "/ingest"` proxies analytics through our own origin, so the - * browser never talks to a third-party analytics host directly. - * - `src/lib/analytics-server.ts:72` derives the server-side id as - * sha256(user-agent + UTC day) and sets `$process_person_profile: false`; - * no IP is stored. - * - `src/lib/rate-limit.ts:21` keeps rate-limit buckets in an in-memory - * Map keyed by IP. It is process-local, never written to disk, and lost - * on every deploy. - * If any of those change, this page changes in the same commit. + * Every figure is read off the running system rather than drafted from a + * template. Two of them corrected the first version, which was wrong: + * + * - It said no IP address is stored. That was true of the server-side + * capture (src/lib/analytics-server.ts sets no IP and disables person + * profiles) and false of the site as a whole: the browser SDK posts + * through /ingest and PostHog records the request IP on ingestion, with + * `anonymize_ips` off on this project. Checked 2026-10-01: an IP is + * present on 100% of events, and PostHog derives country and city from + * it. Understating collection is as wrong as overstating it. + * - It gave no retention period at all. The project's configured + * product-analytics retention is one year. + * + * Verified alongside: `autocapture: false` and `session_recording_opt_in: + * false` on the project, so there is no automatic event capture and no + * session replay. If any of that changes, this page changes in the same + * commit. */ export default function PrivacyPage() { return ( @@ -42,90 +46,165 @@ export default function PrivacyPage() {

Privacy

OpenChainBench has no accounts, no sign-in and no payments. It - measures infrastructure, not people. + measures infrastructure, not people. This page says exactly what is + collected anyway, and what you can switch off.

- +

- We collect anonymous, aggregate usage statistics so we know which - benchmarks are read. We do not ask for your name, your email or your - wallet, we do not sell or share data with advertisers, and we have no - way to work out who you are from what we store. -

- - -

- Page analytics run on PostHog, proxied through this site’s own - origin so your browser never contacts a third-party analytics host - directly. Automatic event capture is switched off: we record a - pageview per route, a page-leave event, and three deliberate - interactions (an outbound link click, a copy, a search). Each visit is - tied to a random identifier stored in your browser, never to an - identity. User profiles are disabled in our configuration and the code - never calls the function that would create one. + Reading the site produces ordinary web analytics. The categories are:

+
    +
  • + Your IP address, recorded by our + analytics processor when a request arrives, and the approximate + location it derives from that: country and city, nothing finer. +
    • +
  • + Your browser and device: the + user-agent string, screen size, and the page performance timings + behind our Core Web Vitals. +
    • +
  • + What you looked at: the pages + you opened, the page you arrived from, and three deliberate actions, + an outbound link click, a copy, and a site search. +
    • +
  • + A random device identifier{" "} + stored in your browser, so two pageviews can be recognised as one + visit. It is tied to no identity, and we never call the function that + would create a user profile. +
    • +

- A small number of machine-readable surfaces are counted on the server - instead, because no browser renders them: the Markdown views,{" "} + Three machine-readable surfaces are counted on the server instead, + because no browser renders them: the Markdown views,{" "} /api/stat and /api/citable. There the - identifier is a daily hash of the requesting user agent. No IP address - is stored. + identifier is a daily hash of the requesting user agent, and no profile + is created. +

+

+ We never ask for and never want: your name, your email, a wallet + address, a private key, a seed phrase or an API key. No feature of this + site requires one. +

+

+ Automatic event capture is off, so nothing is recorded except the + events named above. Session recording is off: no replays, no heatmaps, + no keystroke or form capture. +

+ + +
    +
  • + To know which benchmarks are read, so we work on the ones people use. +
    • +
  • + To measure whether pages load well, which is what the performance + timings are for. +
    • +
  • + To rate limit the public API per address, so one caller cannot deny + the service to everyone else. +
    • +
+

+ Not for advertising, not for profiling, and not for building a picture + of you across other sites. There is no advertising on this site and no + cross-site tracking identifier.

+ + +
    +
  • + PostHog, our analytics + processor, hosted in the United States. Analytics are proxied through + this site’s own origin, so your browser never contacts them + directly. +
    • +
  • + Vercel, which hosts and serves + the site and necessarily handles requests to it. +
    • +

- Public endpoints are rate limited per IP address. Those counters live - in memory in the serving process, are never written to disk, and are - discarded whenever the site is redeployed. + Nobody else. We do not sell data, share it with advertisers or data + brokers, or pass it to any third party beyond those two processors.

- +
    -
  • No accounts, passwords or email addresses, because there is nothing to sign in to.
    • -
  • No wallet addresses, private keys or API keys. Never send us one.
    • -
  • No advertising or cross-site tracking identifiers, and no data sold or shared with data brokers.
    • -
  • No session recordings, heatmaps or form capture.
    • +
  • + Analytics events: one year, the + retention configured on our analytics project, after which they are + deleted by the processor. +
    • +
  • + Rate-limit counters: minutes.{" "} + They live in the memory of the serving process, are never written to + disk, and are discarded whenever the site is redeployed. +
    • +
  • + Benchmark measurements: indefinitely, + because they describe infrastructure providers rather than visitors + and contain no personal data. +
    • +
+ + +
    +
  • + Analytics can be blocked in your browser, or by any content blocker, + or by enabling Do Not Track. Nothing on the site stops working. +
    • +
  • + Clearing your browser storage removes the random device identifier, + and the next visit starts a new one. +
    • +
  • + Because we hold no name, email or account, we cannot look up, export + or delete “your” records on request: there is nothing + tying them to you. If you believe we hold something about you, write + to{" "} + + {SITE.email} + {" "} + and we will look. +

The Model Context Protocol endpoint at{" "} - {SITE.url}/api/mcp/mcp is public, read-only and requires - no authentication or account. It answers questions about published - benchmark data. It stores nothing about the conversation that reached - it: requests are served and dropped, with only the same anonymous - aggregate counting described above. + {SITE.url}/api/mcp/mcp is public, read-only and requires no + authentication or account. It answers questions about published + benchmark data and stores nothing about the conversation that reached + it. A question containing something that looks like a credential is + redacted before it appears anywhere in the response.

- When you reach this data through an AI assistant, that assistant has - its own privacy policy covering your conversation. We receive the - question your assistant chooses to send us and nothing else. + When you reach this data through an AI assistant, that assistant has its + own privacy policy covering your conversation. We receive the question + it chooses to send us and nothing else.

- +

- Benchmark measurements are published openly under CC-BY-4.0. They - describe infrastructure providers, not visitors, and contain no - personal data. Methodology and raw harness code are public in the{" "} - - repository - - . + This is a technical reference for people choosing infrastructure + providers. It is not directed at children, and we do not knowingly + collect anything from them.

- +

- Because we hold no identifying data, we cannot look up, export or - delete “your” records. You can stop the anonymous - analytics at any time by blocking them in your browser or enabling Do - Not Track. If you believe we hold something about you, or you want - anything on this page clarified, write to{" "} + This page changes in public commits like the rest of the site, in the + same change that alters what is collected. Questions and corrections to{" "} {SITE.email} - . -

-

- See also the{" "} + . See also the{" "} terms of use {" "} diff --git a/src/lib/mcp-tools.test.ts b/src/lib/mcp-tools.test.ts index a5e0c9b9c..636225397 100644 --- a/src/lib/mcp-tools.test.ts +++ b/src/lib/mcp-tools.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { compactRow, compareOnBenchmark, matchUseCase, scoreBenchmark, searchBenchmarks } from "./mcp-tools"; +import { compactRow, compareOnBenchmark, matchUseCase, redactSecrets, scoreBenchmark, searchBenchmarks } from "./mcp-tools"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; const SITE = "https://openchainbench.com"; @@ -145,3 +145,45 @@ describe("head to head comparison", () => { expect(c.missing).toEqual(["nobody"]); }); }); + +describe("a query is never echoed back with a secret in it", () => { + // Found by calling production: asked to search for "my API key abc123SECRET", + // the endpoint returned the secret inside its own payload. Nothing stored it, + // but it reached the model's context and whatever the client logs, and the + // rule here is that a key appears in none of those. + test("redacts a named credential", () => { + expect(redactSecrets("test my RPC with my api key abc123SECRETVALUE")).not.toContain("abc123SECRETVALUE"); + }); + + test("redacts a bare high-entropy run", () => { + expect(redactSecrets("use vcp_4Zj1bbjGZf1m4tUxJ52RD4SCW")).toBe("use [redacted]"); + }); + + test("redacts a keyed endpoint", () => { + expect(redactSecrets("https://eth.example.com/v2/deadbeefkey")).toBe("[redacted-url]"); + }); + + test("leaves an ordinary question alone", () => { + expect(redactSecrets("which solana rpc is fastest")).toBe("which solana rpc is fastest"); + expect(redactSecrets("Alchemy or QuickNode on Base")).toBe("Alchemy or QuickNode on Base"); + }); +}); + +describe("weak matches are not published as relevant", () => { + const pool = [ + bench(), + bench({ slug: "buyback-audit", title: "Buyback audit", category: "Trading", results: [r("x", "X", 1)] }), + ]; + + test("a prefix-only hit is below the floor", () => { + // "buy SOL for me" scored buyback-audit and three Solana benches on three + // leading letters. Eight results for a question that matches nothing is + // noise a model may read as relevance. + expect(searchBenchmarks(pool, { query: "buy SOL for me", siteUrl: SITE })).toHaveLength(0); + }); + + test("a real provider-name match still clears it", () => { + const hits = searchBenchmarks(pool, { query: "helius triton", siteUrl: SITE }); + expect(hits.map((h) => h.slug)).toEqual(["solana-rpc"]); + }); +}); diff --git a/src/lib/mcp-tools.ts b/src/lib/mcp-tools.ts index 53ded1746..d38393940 100644 --- a/src/lib/mcp-tools.ts +++ b/src/lib/mcp-tools.ts @@ -112,12 +112,22 @@ export function scoreBenchmark(b: Benchmark, query: string): number { const hay = new Set(haystack(b)); const slugTokens = new Set(tokens(b.slug)); const titleTokens = new Set(tokens(b.title)); + // The category is what the bench is ABOUT, as opposed to which chain it + // happens to cover. Without it a chain name outranks the subject: "cheapest + // bridge to arbitrum" put arbitrum-nova-rpc and arbitrum-rpc above + // bridge-fee, because "arbitrum" hit two slugs while "bridge" hit one. + // Matched as a prefix in both directions so the singular a user types + // ("bridge") reaches the plural we store ("Bridges"). + const categoryTokens = tokens(b.category); let score = 0; for (const t of q) { + const subject = categoryTokens.some((c) => c.startsWith(t) || t.startsWith(c)); + if (subject) score += 3; if (slugTokens.has(t)) score += 4; else if (titleTokens.has(t)) score += 2; else if (hay.has(t)) score += 1; + else if (subject) continue; // Prefix match catches "solana" against "solana-rpc" and "arb" against // "arbitrum", which exact token overlap misses. else if ([...hay].some((h) => h.startsWith(t) || t.startsWith(h))) score += 0.5; @@ -127,6 +137,40 @@ export function scoreBenchmark(b: Benchmark, query: string): number { return score / q.length; } + +/** + * Minimum score for a match to be published. + * + * A slug token is worth 4 and a provider name 1, so a real question clears + * this easily ("solana rpc" scores 4, "helius triton" scores 1). What it cuts + * is prefix-only noise: "buy SOL for me" matched buyback-audit and three + * Solana benches at 0.5, on nothing but the first three letters. Returning + * those invites a model to treat them as relevant. + */ +const MIN_SCORE = 1; + +/** + * Redact anything secret-shaped before echoing a query back. + * + * The search echo is useful (the model should see what it searched) but it is + * also a reflection: asked "test my RPC with my API key abc123SECRET", the + * endpoint returned the secret inside its own payload. Nothing persists it, + * yet it still reaches the model's context and the client's logs, and the + * rule here is that a key never appears in any of those. + * + * Deliberately blunt. Over-redacting a benchmark query costs a reader + * nothing; under-redacting a live credential costs them the credential. + */ +export function redactSecrets(text: string): string { + return text + // key=value and bearer forms + .replace(/\b(api[-_ ]?key|apikey|token|secret|bearer|password|pwd)\b\s*[:=]?\s*\S+/gi, "$1 [redacted]") + // bare high-entropy runs: 16+ chars mixing letters and digits + .replace(/\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{16,}\b/g, "[redacted]") + // anything that looks like a keyed endpoint + .replace(/https?:\/\/\S+/gi, "[redacted-url]"); +} + export type SearchResult = CompactRow & { score: number }; /** @@ -150,7 +194,7 @@ export function searchBenchmarks( const scored = pool .map((b) => ({ b, score: scoreBenchmark(b, opts.query as string) })) - .filter((x) => x.score > 0) + .filter((x) => x.score >= MIN_SCORE) .sort((a, b) => b.score - a.score || a.b.slug.localeCompare(b.b.slug)) .slice(0, limit);