feat(docs): docs MCP at cloudhal.dev/mcp and Accept: text/markdown (RFC-0037 phase 3)

.changeset/mcp-docs-pointer.md

@@ -0,0 +1,4 @@
+---
+"@cloudhal/mcp-server": patch
+---
+The server's instructions point developer agents to the docs at https://cloudhal.dev/llms.txt. The server itself has no docs tools; the docs have their own MCP server at https://cloudhal.dev/mcp.

apps/docs/scripts/postbuild.ts

@@ -5,31 +5,23 @@
  * - `/schemas/v1/<name>.schema.json` at the permanent `$id` URLs, plus `index.json`; the build
  *   fails when a schema's `$id` is not its URL here;
  * - every page as Markdown at `<path>.md` and `<path>/index.md`, links absolute;
- * - `/llms.txt`, `/llms-full.txt` and one pair per section (llmstxt.org).
+ * - `/llms.txt`, `/llms-full.txt` and one pair per section (llmstxt.org);
+ * - the docs MCP's search index (`/_mcp/search-index.json`, read by `src/worker`).
  */
 import { cp, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { allPages, type Page } from "../src/lib/pages.ts";
 import { redirectLines } from "../src/lib/redirects.ts";
 import { ORIGIN } from "../src/lib/repo-docs.ts";
+import { buildSearchIndex, SEARCH_INDEX_PATH } from "../src/lib/search-index.ts";
+import { SECTIONS, sectionOf, urlOf } from "../src/lib/sections.ts";
 
 const APP = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
 const REPO = path.resolve(APP, "../..");
 const SCHEMAS = path.join(REPO, "packages/core-contracts/schemas");
 const DIST = path.join(APP, "dist");
 
-/** Sections with their own `llms.txt`, in index order. */
-const SECTIONS = [
-  { key: "start", prefix: "docs/start/", title: "Start" },
-  { key: "agents", prefix: "docs/agents", title: "Agents and MCP" },
-  { key: "reference", prefix: "docs/reference/", title: "Reference" },
-  { key: "guides", prefix: "docs/guides/", title: "Guides" },
-  { key: "contributing", prefix: "docs/", title: "Contributing and internals" },
-  { key: "adr", prefix: "docs/adr", title: "Architecture decisions" },
-  { key: "rfcs", prefix: "docs/rfcs", title: "Proposals" },
-] as const;
-
 interface SchemaEntry {
   readonly name: string;
   readonly url: string;
@@ -55,8 +47,6 @@ async function copySchemas(): Promise<SchemaEntry[]> {
   return entries;
 }
 
-const urlOf = (page: Page) => (page.id === "index" ? `${ORIGIN}/` : `${ORIGIN}/${page.id}`);
-
 function markdownOf(page: Page): string {
   const front = [
     "---",
@@ -85,15 +75,6 @@ async function writeMarkdown(pages: readonly Page[]): Promise<void> {
   }
 }
 
-function sectionOf(page: Page): (typeof SECTIONS)[number] | undefined {
-  if (!page.id.startsWith("docs/")) return undefined;
-  return SECTIONS.find((s) =>
-    s.key === "contributing"
-      ? !/^docs\/(start|agents|reference|guides|adr|rfcs)(\/|$)/.test(page.id)
-      : page.id === s.prefix || page.id.startsWith(s.prefix),
-  );
-}
-
 const linkLine = (page: Page) => `- [${page.title}](${urlOf(page)}.md): ${page.description}`;
 
 function llmsIndex(pages: readonly Page[], schemas: readonly SchemaEntry[]): string {
@@ -139,4 +120,6 @@ const schemas = await copySchemas();
 const pages = allPages(REPO, true);
 await writeMarkdown(pages);
 await writeLlms(pages, schemas);
+await mkdir(path.dirname(path.join(DIST, SEARCH_INDEX_PATH)), { recursive: true });
+await writeFile(path.join(DIST, SEARCH_INDEX_PATH), JSON.stringify(buildSearchIndex(pages)));
 console.log(`cloudhal.dev: ${pages.length} pages, ${schemas.length} schemas`);

apps/docs/src/lib/search-index.ts

@@ -0,0 +1,62 @@
+/**
+ * Builds the docs MCP's search index from the same page list as the HTML, the `.md` copies and
+ * `llms.txt` (ADR-0039), so search finds exactly the published pages. Build-time only.
+ */
+import type { Page } from "./pages.ts";
+import { type SearchDoc, type SearchIndex, tokens } from "./search.ts";
+
+export { SEARCH_INDEX_PATH } from "./search.ts";
+
+import { sectionOf, urlOf } from "./sections.ts";
+
+/** Field weights: a word in the title counts five times, in the description or path twice. */
+const WEIGHTS = { title: 5, description: 2, path: 2, body: 1 } as const;
+
+/** Markdown to searchable text: links keep their text, markup and HTML tags go. */
+export function plainText(markdown: string): string {
+  return markdown
+    .replace(/<[^>]+>/g, " ")
+    .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+    .replace(/^\s*```.*$/gm, " ")
+    .replace(/^\s*\|?[\s:|-]+\|[\s:|-]*$/gm, " ")
+    .replace(/[`*_#>|]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function buildSearchIndex(pages: readonly Page[]): SearchIndex {
+  const docs: SearchDoc[] = [];
+  const postings = new Map<string, number[]>();
+  pages.forEach((page, doc) => {
+    const text = plainText(page.body);
+    const counts = new Map<string, number>();
+    const fields = [
+      [page.title, WEIGHTS.title],
+      [page.description, WEIGHTS.description],
+      [page.id.replace(/[/-]/g, " "), WEIGHTS.path],
+      [text, WEIGHTS.body],
+    ] as const;
+    let length = 0;
+    for (const [value, weight] of fields) {
+      for (const t of tokens(value)) {
+        counts.set(t, (counts.get(t) ?? 0) + weight);
+        length += weight;
+      }
+    }
+    for (const [t, tf] of counts) {
+      const list = postings.get(t) ?? [];
+      list.push(doc, tf);
+      postings.set(t, list);
+    }
+    const section = sectionOf(page)?.key;
+    docs.push({
+      title: page.title,
+      url: urlOf(page),
+      ...(section ? { section } : {}),
+      text,
+      length,
+    });
+  });
+  return { version: 1, docs, terms: Object.fromEntries(postings) };
+}

apps/docs/src/lib/search.ts

@@ -0,0 +1,151 @@
+/**
+ * Docs search for the docs MCP's `search_docs` (ADR-0039 rule 4). The post-build step writes the
+ * index (`search-index.ts`); the Worker loads it once per isolate and ranks with BM25. Pure and
+ * Worker-safe: no Node imports.
+ */
+
+/** Where the post-build step writes the index and the Worker reads it (through the assets). */
+export const SEARCH_INDEX_PATH = "/_mcp/search-index.json";
+
+export interface SearchDoc {
+  readonly title: string;
+  /** `https://cloudhal.dev/docs/…`, the HTML page; `read_doc` takes it as is. */
+  readonly url: string;
+  /** Section key (`start`, `agents`, …); site pages outside `/docs/` have none. */
+  readonly section?: string;
+  /** Plain text of the page, for snippets. */
+  readonly text: string;
+  /** Weighted token count (BM25's document length). */
+  readonly length: number;
+}
+
+export interface SearchIndex {
+  readonly version: 1;
+  readonly docs: readonly SearchDoc[];
+  /** Term → flat `[doc, weightedFrequency, doc, weightedFrequency, …]`. */
+  readonly terms: Readonly<Record<string, readonly number[]>>;
+}
+
+export interface SearchHit {
+  readonly title: string;
+  readonly url: string;
+  readonly snippet: string;
+}
+
+const STOP = new Set(
+  "a an and are as at be by can do for from has have how i if in into is it its not of on or so that the their then there this to was what when where which who why will with you your".split(
+    " ",
+  ),
+);
+
+/** Lower case, accents dropped, words of letters and digits, a light plural stem. */
+export function stem(word: string): string {
+  if (word.length > 4 && word.endsWith("ies")) return `${word.slice(0, -3)}y`;
+  if (word.length > 3 && word.endsWith("s") && !word.endsWith("ss")) return word.slice(0, -1);
+  return word;
+}
+
+export function tokens(text: string): string[] {
+  const words =
+    text
+      .toLowerCase()
+      .normalize("NFKD")
+      .replace(/\p{M}/gu, "")
+      .match(/[\p{L}\p{N}]+/gu) ?? [];
+  return words.filter((w) => !STOP.has(w)).map(stem);
+}
+
+const K1 = 1.2;
+const B = 0.75;
+/** A query word also matches longer terms it starts with, at this weight. */
+const PREFIX_WEIGHT = 0.4;
+
+/** Each query term with the index terms it matches and their weight. */
+function expand(index: SearchIndex, query: string): Map<string, Map<string, number>> {
+  const out = new Map<string, Map<string, number>>();
+  const all = Object.keys(index.terms);
+  for (const term of new Set(tokens(query))) {
+    const matches = new Map<string, number>();
+    if (index.terms[term]) matches.set(term, 1);
+    if (term.length >= 3) {
+      for (const t of all) if (t !== term && t.startsWith(term)) matches.set(t, PREFIX_WEIGHT);
+    }
+    if (matches.size > 0) out.set(term, matches);
+  }
+  return out;
+}
+
+function score(index: SearchIndex, query: string, section?: string): Map<number, number> {
+  const n = index.docs.length;
+  const avg = index.docs.reduce((sum, d) => sum + d.length, 0) / Math.max(n, 1);
+  const scores = new Map<number, number>();
+  for (const matches of expand(index, query).values()) {
+    // A query word counts once per page: its best-matching term.
+    const best = new Map<number, number>();
+    for (const [term, weight] of matches) {
+      const postings = index.terms[term] ?? [];
+      const df = postings.length / 2;
+      const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+      for (let i = 0; i < postings.length; i += 2) {
+        const doc = postings[i] as number;
+        const tf = postings[i + 1] as number;
+        const d = index.docs[doc];
+        if (!d || (section && d.section !== section)) continue;
+        const s = (weight * idf * tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * d.length) / avg));
+        if (s > (best.get(doc) ?? 0)) best.set(doc, s);
+      }
+    }
+    for (const [doc, s] of best) scores.set(doc, (scores.get(doc) ?? 0) + s);
+  }
+  return scores;
+}
+
+const SNIPPET = 240;
+
+/** Where the most distinct query words fall within one snippet's width (the first 20 hits each). */
+function bestPosition(lower: string, words: readonly string[]): number {
+  const hits: [number, string][] = [];
+  for (const word of words) {
+    for (
+      let at = lower.indexOf(word), n = 0;
+      at >= 0 && n < 20;
+      at = lower.indexOf(word, at + 1), n++
+    ) {
+      hits.push([at, word]);
+    }
+  }
+  let best = { at: 0, count: 0 };
+  for (const [at] of hits) {
+    const near = new Set(hits.filter(([p]) => p >= at && p < at + SNIPPET / 2).map(([, w]) => w));
+    if (near.size > best.count || (near.size === best.count && at < best.at)) {
+      best = { at, count: near.size };
+    }
+  }
+  return best.at;
+}
+
+/** About 240 characters of `text` around the densest cluster of query words. */
+export function snippet(text: string, query: string): string {
+  const at = bestPosition(text.toLowerCase(), [...new Set(tokens(query))]);
+  let start = Math.max(0, at - SNIPPET / 3);
+  if (start > 0) start = text.indexOf(" ", start) + 1 || start;
+  let end = Math.min(text.length, start + SNIPPET);
+  if (end < text.length)
+    end = text.lastIndexOf(" ", end) > start ? text.lastIndexOf(" ", end) : end;
+  return `${start > 0 ? "…" : ""}${text.slice(start, end).trim()}${end < text.length ? "…" : ""}`;
+}
+
+/** The best `limit` pages for `query`, optionally within one section. */
+export function searchDocs(
+  index: SearchIndex,
+  query: string,
+  options: { section?: string | undefined; limit?: number } = {},
+): SearchHit[] {
+  return [...score(index, query, options.section)]
+    .sort((a, b) => b[1] - a[1] || a[0] - b[0])
+    .slice(0, options.limit ?? 8)
+    .flatMap(([doc]) => {
+      const d = index.docs[doc];
+      return d ? [{ title: d.title, url: d.url, snippet: snippet(d.text, query) }] : [];
+    });
+}

apps/docs/src/lib/sections.ts

@@ -0,0 +1,30 @@
+/**
+ * cloudhal.dev's sections for agents (ADR-0039): each has its own `llms.txt`, and `search_docs`
+ * on the docs MCP can be limited to one. Build-time only.
+ */
+import type { Page } from "./pages.ts";
+import { ORIGIN } from "./repo-docs.ts";
+
+/** Sections with their own `llms.txt`, in index order. */
+export const SECTIONS = [
+  { key: "start", prefix: "docs/start/", title: "Start" },
+  { key: "agents", prefix: "docs/agents", title: "Agents and MCP" },
+  { key: "reference", prefix: "docs/reference/", title: "Reference" },
+  { key: "guides", prefix: "docs/guides/", title: "Guides" },
+  { key: "contributing", prefix: "docs/", title: "Contributing and internals" },
+  { key: "adr", prefix: "docs/adr", title: "Architecture decisions" },
+  { key: "rfcs", prefix: "docs/rfcs", title: "Proposals" },
+] as const;
+
+export type Section = (typeof SECTIONS)[number];
+
+export function sectionOf(page: Page): Section | undefined {
+  if (!page.id.startsWith("docs/")) return undefined;
+  return SECTIONS.find((s) =>
+    s.key === "contributing"
+      ? !/^docs\/(start|agents|reference|guides|adr|rfcs)(\/|$)/.test(page.id)
+      : page.id === s.prefix || page.id.startsWith(s.prefix),
+  );
+}
+
+export const urlOf = (page: Page) => (page.id === "index" ? `${ORIGIN}/` : `${ORIGIN}/${page.id}`);