From aae143f2a94f6b40186bcb39f81db67e05f9de89 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:43:36 +0300 Subject: [PATCH] feat(docs): docs MCP at cloudhal.dev/mcp and Accept: text/markdown (RFC-0037 phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cloudhal-dev Worker gets a small script in front of the assets: - /mcp: a standalone, read-only, unauthenticated docs MCP (Streamable HTTP, stateless, JSON answers) with search_docs and read_doc, rate limited to 100 requests a minute per IP (MCP_RATE_LIMITER); - Accept: text/markdown on a page answers its .md copy with Vary: Accept and x-markdown-tokens; everything else goes to the assets binding unchanged. postbuild.ts writes the search index from the same page list as the HTML, the .md copies and llms.txt; the Worker ranks with BM25 and prefix matches. Sections move to src/lib/sections.ts. ADR-0039 rule 4 and RFC-0037 §4 are amended: the site MCP server (cloudhal mcp, remote /mcp) gets no docs tools. It serves agents working on sites, its how-to knowledge lives in skills, tool descriptions and server instructions, and docs tools would add irrelevant tools for narrow keys, a second knowledge source and an outbound dependency. Its instructions now only name https://cloudhal.dev/llms.txt. docs/agents/docs-for-agents.md explains how to add the docs MCP to Claude Code, Codex, Cursor, VS Code and Claude Desktop. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/mcp-docs-pointer.md | 4 + apps/docs/scripts/postbuild.ts | 29 +- apps/docs/src/lib/search-index.ts | 62 +++++ apps/docs/src/lib/search.ts | 151 +++++++++++ apps/docs/src/lib/sections.ts | 30 +++ apps/docs/src/worker/index.ts | 96 +++++++ apps/docs/src/worker/mcp.ts | 226 ++++++++++++++++ apps/docs/src/worker/paths.ts | 40 +++ apps/docs/static/_headers | 4 + apps/docs/test/worker.test.ts | 253 ++++++++++++++++++ apps/docs/tsconfig.json | 2 +- apps/docs/wrangler.jsonc | 12 +- docs/adr/0039-cloudhal-dev-docs-site.md | 4 +- docs/adr/README.md | 2 +- docs/agents/docs-for-agents.md | 52 +++- docs/rfcs/0037-cloudhal-dev-docs-site.md | 16 +- .../mcp-server/src/server-instructions.ts | 4 +- 17 files changed, 948 insertions(+), 39 deletions(-) create mode 100644 .changeset/mcp-docs-pointer.md create mode 100644 apps/docs/src/lib/search-index.ts create mode 100644 apps/docs/src/lib/search.ts create mode 100644 apps/docs/src/lib/sections.ts create mode 100644 apps/docs/src/worker/index.ts create mode 100644 apps/docs/src/worker/mcp.ts create mode 100644 apps/docs/src/worker/paths.ts create mode 100644 apps/docs/test/worker.test.ts diff --git a/.changeset/mcp-docs-pointer.md b/.changeset/mcp-docs-pointer.md new file mode 100644 index 00000000..cfa2768d --- /dev/null +++ b/.changeset/mcp-docs-pointer.md @@ -0,0 +1,4 @@ +--- +"@cloudhal/mcp-server": patch +--- +The server's instructions point developer agents to the docs at https://cloudhal.dev/llms.txt. The server itself has no docs tools; the docs have their own MCP server at https://cloudhal.dev/mcp. diff --git a/apps/docs/scripts/postbuild.ts b/apps/docs/scripts/postbuild.ts index 9636e1d2..f1491986 100644 --- a/apps/docs/scripts/postbuild.ts +++ b/apps/docs/scripts/postbuild.ts @@ -5,7 +5,8 @@ * - `/schemas/v1/.schema.json` at the permanent `$id` URLs, plus `index.json`; the build * fails when a schema's `$id` is not its URL here; * - every page as Markdown at `.md` and `/index.md`, links absolute; - * - `/llms.txt`, `/llms-full.txt` and one pair per section (llmstxt.org). + * - `/llms.txt`, `/llms-full.txt` and one pair per section (llmstxt.org); + * - the docs MCP's search index (`/_mcp/search-index.json`, read by `src/worker`). */ import { cp, mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -13,23 +14,14 @@ import { fileURLToPath } from "node:url"; import { allPages, type Page } from "../src/lib/pages.ts"; import { redirectLines } from "../src/lib/redirects.ts"; import { ORIGIN } from "../src/lib/repo-docs.ts"; +import { buildSearchIndex, SEARCH_INDEX_PATH } from "../src/lib/search-index.ts"; +import { SECTIONS, sectionOf, urlOf } from "../src/lib/sections.ts"; const APP = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const REPO = path.resolve(APP, "../.."); const SCHEMAS = path.join(REPO, "packages/core-contracts/schemas"); const DIST = path.join(APP, "dist"); -/** Sections with their own `llms.txt`, in index order. */ -const SECTIONS = [ - { key: "start", prefix: "docs/start/", title: "Start" }, - { key: "agents", prefix: "docs/agents", title: "Agents and MCP" }, - { key: "reference", prefix: "docs/reference/", title: "Reference" }, - { key: "guides", prefix: "docs/guides/", title: "Guides" }, - { key: "contributing", prefix: "docs/", title: "Contributing and internals" }, - { key: "adr", prefix: "docs/adr", title: "Architecture decisions" }, - { key: "rfcs", prefix: "docs/rfcs", title: "Proposals" }, -] as const; - interface SchemaEntry { readonly name: string; readonly url: string; @@ -55,8 +47,6 @@ async function copySchemas(): Promise { return entries; } -const urlOf = (page: Page) => (page.id === "index" ? `${ORIGIN}/` : `${ORIGIN}/${page.id}`); - function markdownOf(page: Page): string { const front = [ "---", @@ -85,15 +75,6 @@ async function writeMarkdown(pages: readonly Page[]): Promise { } } -function sectionOf(page: Page): (typeof SECTIONS)[number] | undefined { - if (!page.id.startsWith("docs/")) return undefined; - return SECTIONS.find((s) => - s.key === "contributing" - ? !/^docs\/(start|agents|reference|guides|adr|rfcs)(\/|$)/.test(page.id) - : page.id === s.prefix || page.id.startsWith(s.prefix), - ); -} - const linkLine = (page: Page) => `- [${page.title}](${urlOf(page)}.md): ${page.description}`; function llmsIndex(pages: readonly Page[], schemas: readonly SchemaEntry[]): string { @@ -139,4 +120,6 @@ const schemas = await copySchemas(); const pages = allPages(REPO, true); await writeMarkdown(pages); await writeLlms(pages, schemas); +await mkdir(path.dirname(path.join(DIST, SEARCH_INDEX_PATH)), { recursive: true }); +await writeFile(path.join(DIST, SEARCH_INDEX_PATH), JSON.stringify(buildSearchIndex(pages))); console.log(`cloudhal.dev: ${pages.length} pages, ${schemas.length} schemas`); diff --git a/apps/docs/src/lib/search-index.ts b/apps/docs/src/lib/search-index.ts new file mode 100644 index 00000000..48751445 --- /dev/null +++ b/apps/docs/src/lib/search-index.ts @@ -0,0 +1,62 @@ +/** + * Builds the docs MCP's search index from the same page list as the HTML, the `.md` copies and + * `llms.txt` (ADR-0039), so search finds exactly the published pages. Build-time only. + */ +import type { Page } from "./pages.ts"; +import { type SearchDoc, type SearchIndex, tokens } from "./search.ts"; + +export { SEARCH_INDEX_PATH } from "./search.ts"; + +import { sectionOf, urlOf } from "./sections.ts"; + +/** Field weights: a word in the title counts five times, in the description or path twice. */ +const WEIGHTS = { title: 5, description: 2, path: 2, body: 1 } as const; + +/** Markdown to searchable text: links keep their text, markup and HTML tags go. */ +export function plainText(markdown: string): string { + return markdown + .replace(/<[^>]+>/g, " ") + .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/^\s*```.*$/gm, " ") + .replace(/^\s*\|?[\s:|-]+\|[\s:|-]*$/gm, " ") + .replace(/[`*_#>|]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +export function buildSearchIndex(pages: readonly Page[]): SearchIndex { + const docs: SearchDoc[] = []; + const postings = new Map(); + pages.forEach((page, doc) => { + const text = plainText(page.body); + const counts = new Map(); + const fields = [ + [page.title, WEIGHTS.title], + [page.description, WEIGHTS.description], + [page.id.replace(/[/-]/g, " "), WEIGHTS.path], + [text, WEIGHTS.body], + ] as const; + let length = 0; + for (const [value, weight] of fields) { + for (const t of tokens(value)) { + counts.set(t, (counts.get(t) ?? 0) + weight); + length += weight; + } + } + for (const [t, tf] of counts) { + const list = postings.get(t) ?? []; + list.push(doc, tf); + postings.set(t, list); + } + const section = sectionOf(page)?.key; + docs.push({ + title: page.title, + url: urlOf(page), + ...(section ? { section } : {}), + text, + length, + }); + }); + return { version: 1, docs, terms: Object.fromEntries(postings) }; +} diff --git a/apps/docs/src/lib/search.ts b/apps/docs/src/lib/search.ts new file mode 100644 index 00000000..48c411a2 --- /dev/null +++ b/apps/docs/src/lib/search.ts @@ -0,0 +1,151 @@ +/** + * Docs search for the docs MCP's `search_docs` (ADR-0039 rule 4). The post-build step writes the + * index (`search-index.ts`); the Worker loads it once per isolate and ranks with BM25. Pure and + * Worker-safe: no Node imports. + */ + +/** Where the post-build step writes the index and the Worker reads it (through the assets). */ +export const SEARCH_INDEX_PATH = "/_mcp/search-index.json"; + +export interface SearchDoc { + readonly title: string; + /** `https://cloudhal.dev/docs/…`, the HTML page; `read_doc` takes it as is. */ + readonly url: string; + /** Section key (`start`, `agents`, …); site pages outside `/docs/` have none. */ + readonly section?: string; + /** Plain text of the page, for snippets. */ + readonly text: string; + /** Weighted token count (BM25's document length). */ + readonly length: number; +} + +export interface SearchIndex { + readonly version: 1; + readonly docs: readonly SearchDoc[]; + /** Term → flat `[doc, weightedFrequency, doc, weightedFrequency, …]`. */ + readonly terms: Readonly>; +} + +export interface SearchHit { + readonly title: string; + readonly url: string; + readonly snippet: string; +} + +const STOP = new Set( + "a an and are as at be by can do for from has have how i if in into is it its not of on or so that the their then there this to was what when where which who why will with you your".split( + " ", + ), +); + +/** Lower case, accents dropped, words of letters and digits, a light plural stem. */ +export function stem(word: string): string { + if (word.length > 4 && word.endsWith("ies")) return `${word.slice(0, -3)}y`; + if (word.length > 3 && word.endsWith("s") && !word.endsWith("ss")) return word.slice(0, -1); + return word; +} + +export function tokens(text: string): string[] { + const words = + text + .toLowerCase() + .normalize("NFKD") + .replace(/\p{M}/gu, "") + .match(/[\p{L}\p{N}]+/gu) ?? []; + return words.filter((w) => !STOP.has(w)).map(stem); +} + +const K1 = 1.2; +const B = 0.75; +/** A query word also matches longer terms it starts with, at this weight. */ +const PREFIX_WEIGHT = 0.4; + +/** Each query term with the index terms it matches and their weight. */ +function expand(index: SearchIndex, query: string): Map> { + const out = new Map>(); + const all = Object.keys(index.terms); + for (const term of new Set(tokens(query))) { + const matches = new Map(); + if (index.terms[term]) matches.set(term, 1); + if (term.length >= 3) { + for (const t of all) if (t !== term && t.startsWith(term)) matches.set(t, PREFIX_WEIGHT); + } + if (matches.size > 0) out.set(term, matches); + } + return out; +} + +function score(index: SearchIndex, query: string, section?: string): Map { + const n = index.docs.length; + const avg = index.docs.reduce((sum, d) => sum + d.length, 0) / Math.max(n, 1); + const scores = new Map(); + for (const matches of expand(index, query).values()) { + // A query word counts once per page: its best-matching term. + const best = new Map(); + for (const [term, weight] of matches) { + const postings = index.terms[term] ?? []; + const df = postings.length / 2; + const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5)); + for (let i = 0; i < postings.length; i += 2) { + const doc = postings[i] as number; + const tf = postings[i + 1] as number; + const d = index.docs[doc]; + if (!d || (section && d.section !== section)) continue; + const s = (weight * idf * tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * d.length) / avg)); + if (s > (best.get(doc) ?? 0)) best.set(doc, s); + } + } + for (const [doc, s] of best) scores.set(doc, (scores.get(doc) ?? 0) + s); + } + return scores; +} + +const SNIPPET = 240; + +/** Where the most distinct query words fall within one snippet's width (the first 20 hits each). */ +function bestPosition(lower: string, words: readonly string[]): number { + const hits: [number, string][] = []; + for (const word of words) { + for ( + let at = lower.indexOf(word), n = 0; + at >= 0 && n < 20; + at = lower.indexOf(word, at + 1), n++ + ) { + hits.push([at, word]); + } + } + let best = { at: 0, count: 0 }; + for (const [at] of hits) { + const near = new Set(hits.filter(([p]) => p >= at && p < at + SNIPPET / 2).map(([, w]) => w)); + if (near.size > best.count || (near.size === best.count && at < best.at)) { + best = { at, count: near.size }; + } + } + return best.at; +} + +/** About 240 characters of `text` around the densest cluster of query words. */ +export function snippet(text: string, query: string): string { + const at = bestPosition(text.toLowerCase(), [...new Set(tokens(query))]); + let start = Math.max(0, at - SNIPPET / 3); + if (start > 0) start = text.indexOf(" ", start) + 1 || start; + let end = Math.min(text.length, start + SNIPPET); + if (end < text.length) + end = text.lastIndexOf(" ", end) > start ? text.lastIndexOf(" ", end) : end; + return `${start > 0 ? "…" : ""}${text.slice(start, end).trim()}${end < text.length ? "…" : ""}`; +} + +/** The best `limit` pages for `query`, optionally within one section. */ +export function searchDocs( + index: SearchIndex, + query: string, + options: { section?: string | undefined; limit?: number } = {}, +): SearchHit[] { + return [...score(index, query, options.section)] + .sort((a, b) => b[1] - a[1] || a[0] - b[0]) + .slice(0, options.limit ?? 8) + .flatMap(([doc]) => { + const d = index.docs[doc]; + return d ? [{ title: d.title, url: d.url, snippet: snippet(d.text, query) }] : []; + }); +} diff --git a/apps/docs/src/lib/sections.ts b/apps/docs/src/lib/sections.ts new file mode 100644 index 00000000..317b3cd6 --- /dev/null +++ b/apps/docs/src/lib/sections.ts @@ -0,0 +1,30 @@ +/** + * cloudhal.dev's sections for agents (ADR-0039): each has its own `llms.txt`, and `search_docs` + * on the docs MCP can be limited to one. Build-time only. + */ +import type { Page } from "./pages.ts"; +import { ORIGIN } from "./repo-docs.ts"; + +/** Sections with their own `llms.txt`, in index order. */ +export const SECTIONS = [ + { key: "start", prefix: "docs/start/", title: "Start" }, + { key: "agents", prefix: "docs/agents", title: "Agents and MCP" }, + { key: "reference", prefix: "docs/reference/", title: "Reference" }, + { key: "guides", prefix: "docs/guides/", title: "Guides" }, + { key: "contributing", prefix: "docs/", title: "Contributing and internals" }, + { key: "adr", prefix: "docs/adr", title: "Architecture decisions" }, + { key: "rfcs", prefix: "docs/rfcs", title: "Proposals" }, +] as const; + +export type Section = (typeof SECTIONS)[number]; + +export function sectionOf(page: Page): Section | undefined { + if (!page.id.startsWith("docs/")) return undefined; + return SECTIONS.find((s) => + s.key === "contributing" + ? !/^docs\/(start|agents|reference|guides|adr|rfcs)(\/|$)/.test(page.id) + : page.id === s.prefix || page.id.startsWith(s.prefix), + ); +} + +export const urlOf = (page: Page) => (page.id === "index" ? `${ORIGIN}/` : `${ORIGIN}/${page.id}`); diff --git a/apps/docs/src/worker/index.ts b/apps/docs/src/worker/index.ts new file mode 100644 index 00000000..1a2aff79 --- /dev/null +++ b/apps/docs/src/worker/index.ts @@ -0,0 +1,96 @@ +/** + * The `cloudhal-dev` Worker (ADR-0037, ADR-0039): a small script in front of the static assets. + * + * - `/mcp`: the read-only docs MCP (`mcp.ts`), rate limited per IP; + * - `Accept: text/markdown` on a page answers its Markdown copy, with `Vary: Accept` and an + * `x-markdown-tokens` estimate (RFC-0037 §4); + * - everything else goes to the assets binding unchanged. + */ +import { SEARCH_INDEX_PATH, type SearchIndex } from "../lib/search.ts"; +import { CORS, handleMcp } from "./mcp.ts"; +import { isPagePath, markdownPath, pageId } from "./paths.ts"; + +/** The parts of the Workers bindings this script uses. */ +export interface Env { + readonly ASSETS: { fetch(request: Request): Promise }; + readonly MCP_RATE_LIMITER?: { limit(options: { key: string }): Promise<{ success: boolean }> }; +} + +let index: Promise | undefined; + +function loadIndex(env: Env, origin: string): Promise { + index ??= env.ASSETS.fetch(new Request(new URL(SEARCH_INDEX_PATH, origin))).then((res) => { + if (!res.ok) throw new Error(`search index: ${res.status}`); + return res.json() as Promise; + }); + // A failed load is retried on the next call instead of staying cached. + index.catch(() => { + index = undefined; + }); + return index; +} + +async function markdownAsset(env: Env, origin: string, path: string): Promise { + const res = await env.ASSETS.fetch(new Request(new URL(path, origin))); + return res.status === 200 ? res.text() : undefined; +} + +/** True when the client asks for Markdown (`text/markdown` in `Accept`, not with `q=0`). */ +export function wantsMarkdown(accept: string | null): boolean { + return (accept ?? "").split(",").some((part) => { + const [type, ...params] = part.trim().toLowerCase().split(";"); + if (type?.trim() !== "text/markdown") return false; + const q = params.map((p) => p.trim()).find((p) => p.startsWith("q=")); + return q === undefined || Number(q.slice(2)) > 0; + }); +} + +function withVary(res: Response): Response { + const out = new Response(res.body, res); + out.headers.append("vary", "Accept"); + return out; +} + +async function negotiate(request: Request, env: Env, url: URL): Promise { + const id = wantsMarkdown(request.headers.get("accept")) ? pageId(url.pathname) : undefined; + if (id) { + const res = await env.ASSETS.fetch(new Request(new URL(markdownPath(id), url.origin))); + if (res.status === 200) { + const body = await res.text(); + const headers = new Headers(res.headers); + headers.set("content-type", "text/markdown; charset=utf-8"); + headers.set("x-markdown-tokens", String(Math.ceil(body.length / 4))); + headers.append("vary", "Accept"); + return new Response(request.method === "HEAD" ? null : body, { status: 200, headers }); + } + } + return withVary(await env.ASSETS.fetch(request)); +} + +async function mcp(request: Request, env: Env, url: URL): Promise { + if (request.method === "POST" && env.MCP_RATE_LIMITER) { + const key = request.headers.get("cf-connecting-ip") ?? "unknown"; + const { success } = await env.MCP_RATE_LIMITER.limit({ key }); + if (!success) { + return new Response("Too many requests", { + status: 429, + headers: { "retry-after": "60", ...CORS }, + }); + } + } + return handleMcp(request, { + index: () => loadIndex(env, url.origin), + markdown: (path) => markdownAsset(env, url.origin, path), + selfHost: url.host, + }); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname === "/mcp" || url.pathname === "/mcp/") return mcp(request, env, url); + const page = + (request.method === "GET" || request.method === "HEAD") && isPagePath(url.pathname); + return page ? negotiate(request, env, url) : env.ASSETS.fetch(request); + }, +}; diff --git a/apps/docs/src/worker/mcp.ts b/apps/docs/src/worker/mcp.ts new file mode 100644 index 00000000..075c1ab0 --- /dev/null +++ b/apps/docs/src/worker/mcp.ts @@ -0,0 +1,226 @@ +/** + * The docs MCP at `https://cloudhal.dev/mcp` (ADR-0039 rule 4): Streamable HTTP, stateless, + * read-only, no authentication. Every POST carries one JSON-RPC message (or a batch, for + * 2025-03-26 clients) and gets one JSON answer; there are no sessions and no server-sent events. + * Separate from Cloudhal's site MCP (`cloudhal mcp`, remote `/mcp` on a deployment). + */ +import { type SearchIndex, searchDocs } from "../lib/search.ts"; +import { markdownPath, pageId } from "./paths.ts"; + +export interface McpDeps { + /** The search index, loaded once per isolate. */ + readonly index: () => Promise; + /** A page's Markdown by its `.md` path, or `undefined` when there is none. */ + readonly markdown: (path: string) => Promise; + /** The Worker's own host, accepted by `read_doc` besides cloudhal.dev. */ + readonly selfHost?: string; +} + +const PROTOCOL_VERSIONS = ["2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"]; +const SECTION_KEYS = ["start", "agents", "reference", "guides", "contributing", "adr", "rfcs"]; +const MAX_BODY = 64 * 1024; + +const INSTRUCTIONS = + "Cloudhal's public documentation (https://cloudhal.dev), read-only. Use search_docs to find pages, then read_doc to read one as Markdown. The index of every page is https://cloudhal.dev/llms.txt. This server only reads the docs: to work on a Cloudhal site, connect that site's own MCP server (cloudhal mcp, or /mcp on a deployment; see https://cloudhal.dev/docs/agents/clients)."; + +const READ_ONLY = { readOnlyHint: true, idempotentHint: true, openWorldHint: false }; + +const TOOLS = [ + { + name: "search_docs", + title: "Search the Cloudhal docs", + description: + "Search Cloudhal's documentation (guides, reference, agents and MCP setup, ADRs and RFCs). Returns up to 8 pages with title, url and a snippet; read one with read_doc.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", minLength: 1, maxLength: 200, description: "Words to look for." }, + section: { + type: "string", + enum: SECTION_KEYS, + description: + "Only this section: start, agents (agents and MCP), reference, guides, contributing (internals), adr (decisions) or rfcs (proposals).", + }, + }, + required: ["query"], + additionalProperties: false, + }, + outputSchema: { + type: "object", + properties: { + results: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + url: { type: "string" }, + snippet: { type: "string" }, + }, + required: ["title", "url", "snippet"], + }, + }, + }, + required: ["results"], + }, + annotations: READ_ONLY, + }, + { + name: "read_doc", + title: "Read a Cloudhal docs page", + description: + "One page of Cloudhal's documentation as Markdown, with frontmatter (title, description, url, source). Takes a https://cloudhal.dev URL or a path such as /docs/agents/clients.", + inputSchema: { + type: "object", + properties: { + url: { + type: "string", + minLength: 1, + maxLength: 500, + description: "A https://cloudhal.dev page URL or path (from search_docs or llms.txt).", + }, + }, + required: ["url"], + additionalProperties: false, + }, + annotations: READ_ONLY, + }, +] as const; + +type Id = string | number | null; +interface Message { + readonly jsonrpc?: unknown; + readonly id?: unknown; + readonly method?: unknown; + readonly params?: unknown; +} + +class RpcError extends Error { + readonly code: number; + constructor(code: number, message: string) { + super(message); + this.code = code; + } +} + +const text = (value: string, isError = false) => ({ + content: [{ type: "text", text: value }], + ...(isError ? { isError: true } : {}), +}); + +const record = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + +async function searchTool(args: Record, deps: McpDeps) { + const { query, section } = args; + if (typeof query !== "string" || query.trim() === "" || query.length > 200) { + return text("query must be 1 to 200 characters.", true); + } + if (section !== undefined && (typeof section !== "string" || !SECTION_KEYS.includes(section))) { + return text(`section must be one of ${SECTION_KEYS.join(", ")}.`, true); + } + const results = searchDocs(await deps.index(), query, { section, limit: 8 }); + return { ...text(JSON.stringify({ results })), structuredContent: { results } }; +} + +async function readTool(args: Record, deps: McpDeps) { + const { url } = args; + if (typeof url !== "string" || url.length > 500) return text("url must be a string.", true); + const id = pageId(url, deps.selfHost); + if (!id) return text("read_doc reads only cloudhal.dev pages (a URL or a /docs/… path).", true); + const markdown = await deps.markdown(markdownPath(id)); + if (markdown === undefined) { + return text(`No page at ${url}. Find pages with search_docs or in /llms.txt.`, true); + } + return text(markdown); +} + +const TOOL_HANDLERS: Record = { + search_docs: searchTool, + read_doc: readTool, +}; + +const METHODS: Record, deps: McpDeps) => unknown> = { + initialize: (params) => { + const asked = params.protocolVersion; + const protocolVersion = + typeof asked === "string" && PROTOCOL_VERSIONS.includes(asked) ? asked : PROTOCOL_VERSIONS[0]; + return { + protocolVersion, + capabilities: { tools: { listChanged: false } }, + serverInfo: { name: "cloudhal-docs", title: "Cloudhal docs", version: "1.0.0" }, + instructions: INSTRUCTIONS, + }; + }, + ping: () => ({}), + "tools/list": () => ({ tools: TOOLS }), + "tools/call": (params, deps) => { + const handler = typeof params.name === "string" ? TOOL_HANDLERS[params.name] : undefined; + if (!handler) throw new RpcError(-32602, `Unknown tool: ${String(params.name)}`); + return handler(record(params.arguments), deps); + }, +}; + +/** One JSON-RPC message's answer; `undefined` for notifications and responses. */ +async function answer(message: unknown, deps: McpDeps): Promise { + const m = record(message) as Message; + const id: Id = typeof m.id === "string" || typeof m.id === "number" ? m.id : null; + if (m.jsonrpc !== "2.0" || (typeof m.method !== "string" && m.id === undefined)) { + return { jsonrpc: "2.0", id, error: { code: -32600, message: "Invalid request" } }; + } + if (typeof m.method !== "string" || m.id === undefined) return undefined; + const method = Object.hasOwn(METHODS, m.method) ? METHODS[m.method] : undefined; + if (!method) return { jsonrpc: "2.0", id, error: { code: -32601, message: "Method not found" } }; + try { + return { jsonrpc: "2.0", id, result: await method(record(m.params), deps) }; + } catch (error) { + const code = error instanceof RpcError ? error.code : -32603; + const message = error instanceof RpcError ? error.message : "Internal error"; + return { jsonrpc: "2.0", id, error: { code, message } }; + } +} + +export const CORS = { + "access-control-allow-origin": "*", + "access-control-allow-methods": "POST, OPTIONS", + "access-control-allow-headers": + "content-type, accept, mcp-protocol-version, mcp-session-id, last-event-id", + "access-control-max-age": "86400", +}; + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json", "cache-control": "no-store", ...CORS }, + }); + +const rpcError = (code: number, message: string, status: number) => + json({ jsonrpc: "2.0", id: null, error: { code, message } }, status); + +/** `/mcp`: POST is JSON-RPC; GET (no event stream) and DELETE (no sessions) answer 405. */ +export async function handleMcp(request: Request, deps: McpDeps): Promise { + if (request.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS }); + if (request.method !== "POST") { + return new Response("Method not allowed", { status: 405, headers: { allow: "POST", ...CORS } }); + } + if (!(request.headers.get("content-type") ?? "").includes("application/json")) { + return rpcError(-32700, "Content-Type must be application/json", 415); + } + const raw = await request.text(); + if (raw.length > MAX_BODY) return rpcError(-32600, "Request too large", 413); + let body: unknown; + try { + body = JSON.parse(raw); + } catch { + return rpcError(-32700, "Parse error", 400); + } + if (Array.isArray(body)) { + if (body.length === 0 || body.length > 20) return rpcError(-32600, "Invalid request", 400); + const answers = (await Promise.all(body.map((m) => answer(m, deps)))).filter(Boolean); + return answers.length > 0 ? json(answers) : new Response(null, { status: 202, headers: CORS }); + } + const one = await answer(body, deps); + return one ? json(one) : new Response(null, { status: 202, headers: CORS }); +} diff --git a/apps/docs/src/worker/paths.ts b/apps/docs/src/worker/paths.ts new file mode 100644 index 00000000..fa526857 --- /dev/null +++ b/apps/docs/src/worker/paths.ts @@ -0,0 +1,40 @@ +/** Page URLs to the Markdown copies the post-build step writes (`.md`, `index.md` for `/`). */ + +const HOSTS = new Set(["cloudhal.dev", "www.cloudhal.dev"]); +const ID = /^[a-z0-9][a-z0-9._/-]*$/; + +/** + * A page id from a cloudhal.dev URL or a site path: `https://cloudhal.dev/docs/agents.md#x`, + * `/docs/agents/` and `docs/agents` all give `docs/agents`; `/` gives `index`. Anything on another + * host, or not a plain page path, gives `undefined`. `selfHost` also accepts the Worker's own host + * (`localhost` under `wrangler dev`). + */ +export function pageId(input: string, selfHost?: string): string | undefined { + let value = input.trim(); + if (/^[a-z][a-z0-9+.-]*:/i.test(value) || value.startsWith("//")) { + let url: URL; + try { + url = new URL(value.startsWith("//") ? `https:${value}` : value); + } catch { + return undefined; + } + if (url.protocol !== "https:" && url.protocol !== "http:") return undefined; + if (!HOSTS.has(url.hostname) && url.host !== selfHost) return undefined; + value = url.pathname; + } + const path = value + .replace(/[?#].*$/, "") + .replace(/^\/+/, "") + .replace(/(^|\/)index\.md$/, "") + .replace(/\.md$/, "") + .replace(/\/+$/, "") + .toLowerCase(); + if (path === "" || path === "index") return "index"; + if (!ID.test(path) || path.split("/").some((s) => s === "." || s === "..")) return undefined; + return path; +} + +export const markdownPath = (id: string) => (id === "index" ? "/index.md" : `/${id}.md`); + +/** A path that may be a page: no file extension in its last segment. */ +export const isPagePath = (pathname: string) => !/\.[^/]*$/.test(pathname); diff --git a/apps/docs/static/_headers b/apps/docs/static/_headers index 5ad367ee..da74a5a3 100644 --- a/apps/docs/static/_headers +++ b/apps/docs/static/_headers @@ -21,3 +21,7 @@ /*.md Access-Control-Allow-Origin: * + +# The docs MCP's search index (read by the Worker, not a public API). +/_mcp/* + X-Robots-Tag: noindex diff --git a/apps/docs/test/worker.test.ts b/apps/docs/test/worker.test.ts new file mode 100644 index 00000000..8c76a29c --- /dev/null +++ b/apps/docs/test/worker.test.ts @@ -0,0 +1,253 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { allPages, type Page } from "../src/lib/pages.ts"; +import { SEARCH_INDEX_PATH, searchDocs, snippet, tokens } from "../src/lib/search.ts"; +import { buildSearchIndex, plainText } from "../src/lib/search-index.ts"; +import worker, { type Env, wantsMarkdown } from "../src/worker/index.ts"; +import { isPagePath, markdownPath, pageId } from "../src/worker/paths.ts"; + +const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); + +const page = (id: string, title: string, body: string, description = ""): Page => ({ + id, + title, + description, + body, +}); + +const PAGES = [ + page("index", "Cloudhal", "Agent-native sites."), + page("docs/agents/clients", "Connect an agent", "Claude Code, Codex and Cursor setup for MCP."), + page("docs/guides/search", "Site search", "Full-text search for a site with FTS5 indexes."), + page("docs/adr/0011-mcp", "ADR-0011: MCP tools", "Tools and transports of the MCP server."), +]; + +/** A fake assets binding: the built index plus Markdown and HTML per path. */ +function env(files: Record = {}, limited = false): Env & { seen: string[] } { + const all: Record = { + [SEARCH_INDEX_PATH]: JSON.stringify(buildSearchIndex(PAGES)), + "/index.md": "# Cloudhal\n", + "/docs/agents/clients.md": "---\ntitle: Connect an agent\n---\n# Connect an agent\n", + "/docs/agents/clients": "clients", + ...files, + }; + const seen: string[] = []; + return { + seen, + ASSETS: { + fetch: async (request: Request) => { + const p = new URL(request.url).pathname; + seen.push(p); + const body = all[p]; + return body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body, { headers: { "content-type": "text/html", "x-asset": p } }); + }, + }, + MCP_RATE_LIMITER: { limit: async () => ({ success: !limited }) }, + }; +} + +async function rpc(e: Env, body: unknown, headers: Record = {}) { + const res = await worker.fetch( + new Request("https://cloudhal.dev/mcp", { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(body), + }), + e, + ); + // biome-ignore lint/suspicious/noExplicitAny: JSON-RPC answers, checked field by field below + const json: any = res.status === 200 ? await res.json() : undefined; + return { res, json }; +} + +const call = (e: Env, name: string, args: unknown) => + rpc(e, { jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }); + +describe("search", () => { + it("tokenizes without stop words, with a light plural stem", () => { + expect(tokens("How do the Pages search_docs?")).toEqual(["page", "search", "doc"]); + }); + + it("strips Markdown to plain text", () => { + expect(plainText("## A [link](/x) and `code`\n\n| a | b |\n|---|---|\n")).toBe( + "A link and code a b", + ); + }); + + it("ranks title matches first, matches prefixes, and filters by section", () => { + const index = buildSearchIndex(PAGES); + expect(searchDocs(index, "search")[0]?.url).toBe("https://cloudhal.dev/docs/guides/search"); + expect(searchDocs(index, "curs").map((h) => h.title)).toEqual(["Connect an agent"]); + expect(searchDocs(index, "mcp", { section: "adr" }).map((h) => h.title)).toEqual([ + "ADR-0011: MCP tools", + ]); + expect(searchDocs(index, "zzzz")).toEqual([]); + }); + + it("cuts a snippet around the first match", () => { + const text = `${"word ".repeat(100)}needle ${"tail ".repeat(100)}`; + const s = snippet(text, "needle"); + expect(s).toContain("needle"); + expect(s.startsWith("…") && s.endsWith("…")).toBe(true); + expect(s.length).toBeLessThan(260); + const cluster = `alpha ${"x ".repeat(200)}alpha beta ${"y ".repeat(200)}`; + expect(snippet(cluster, "alpha beta")).toContain("alpha beta"); + }); + + it("finds the client setup page in the real docs", () => { + const index = buildSearchIndex(allPages(REPO, true)); + const hits = searchDocs(index, "connect Claude Code MCP client"); + expect(hits.length).toBeLessThanOrEqual(8); + expect(hits.slice(0, 3).map((h) => h.url)).toContain( + "https://cloudhal.dev/docs/agents/clients", + ); + for (const hit of searchDocs(index, "rollback", { section: "rfcs" })) { + expect(hit.url).toMatch(/^https:\/\/cloudhal\.dev\/docs\/rfcs/); + } + }); +}); + +describe("pageId", () => { + it("accepts cloudhal.dev URLs and site paths", () => { + expect(pageId("https://cloudhal.dev/docs/agents.md#modes")).toBe("docs/agents"); + expect(pageId("/docs/agents/index.md")).toBe("docs/agents"); + expect(pageId("docs/Agents/?x=1")).toBe("docs/agents"); + expect(pageId("https://cloudhal.dev/")).toBe("index"); + expect(pageId("http://localhost:8787/docs", "localhost:8787")).toBe("docs"); + }); + + it("rejects other hosts, schemes and odd paths", () => { + expect(pageId("https://example.com/docs")).toBeUndefined(); + expect(pageId("//evil.test/docs")).toBeUndefined(); + expect(pageId("file:///etc/passwd")).toBeUndefined(); + expect(pageId("/docs/../secrets")).toBeUndefined(); + expect(pageId("/docs/%2e%2e/x")).toBeUndefined(); + }); + + it("maps ids to Markdown paths and spots page paths", () => { + expect(markdownPath("index")).toBe("/index.md"); + expect(markdownPath("docs/agents")).toBe("/docs/agents.md"); + expect(isPagePath("/docs/agents")).toBe(true); + expect(isPagePath("/llms.txt")).toBe(false); + }); +}); + +describe("docs MCP", () => { + it("initializes with the client's protocol version and lists two read-only tools", async () => { + const e = env(); + const init = await rpc(e, { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "t" } }, + }); + expect(init.json.result.protocolVersion).toBe("2025-06-18"); + expect(init.json.result.capabilities).toEqual({ tools: { listChanged: false } }); + expect(init.res.headers.get("mcp-session-id")).toBeNull(); + const old = await rpc(e, { jsonrpc: "2.0", id: 2, method: "initialize", params: {} }); + expect(old.json.result.protocolVersion).toBe("2025-11-25"); + const list = await rpc(e, { jsonrpc: "2.0", id: 3, method: "tools/list" }); + const tools = list.json.result.tools; + expect(tools.map((t: { name: string }) => t.name)).toEqual(["search_docs", "read_doc"]); + expect( + tools.every((t: { annotations: { readOnlyHint: boolean } }) => t.annotations.readOnlyHint), + ).toBe(true); + }); + + it("answers notifications with 202 and unknown methods with -32601", async () => { + const e = env(); + const note = await rpc(e, { jsonrpc: "2.0", method: "notifications/initialized" }); + expect(note.res.status).toBe(202); + const unknown = await rpc(e, { jsonrpc: "2.0", id: 9, method: "resources/list" }); + expect(unknown.json.error.code).toBe(-32601); + const batch = await rpc(e, [ + { jsonrpc: "2.0", id: 1, method: "ping" }, + { jsonrpc: "2.0", method: "notifications/initialized" }, + ]); + expect(batch.json).toEqual([{ jsonrpc: "2.0", id: 1, result: {} }]); + }); + + it("searches, with structured content", async () => { + const { json } = await call(env(), "search_docs", { query: "codex" }); + expect(json.result.structuredContent.results[0]).toMatchObject({ + title: "Connect an agent", + url: "https://cloudhal.dev/docs/agents/clients", + }); + expect(JSON.parse(json.result.content[0].text)).toEqual(json.result.structuredContent); + const bad = await call(env(), "search_docs", { query: "x", section: "nope" }); + expect(bad.json.result.isError).toBe(true); + }); + + it("reads cloudhal.dev pages only", async () => { + const ok = await call(env(), "read_doc", { url: "https://cloudhal.dev/docs/agents/clients" }); + expect(ok.json.result.content[0].text).toContain("# Connect an agent"); + const elsewhere = await call(env(), "read_doc", { url: "https://example.com/docs" }); + expect(elsewhere.json.result.isError).toBe(true); + const missing = await call(env(), "read_doc", { url: "/docs/nope" }); + expect(missing.json.result.content[0].text).toContain("No page at /docs/nope"); + const unknown = await call(env(), "write_doc", {}); + expect(unknown.json.error.code).toBe(-32602); + }); + + it("refuses GET, non-JSON bodies and bad JSON, and rate limits", async () => { + const get = await worker.fetch(new Request("https://cloudhal.dev/mcp"), env()); + expect(get.status).toBe(405); + const form = await worker.fetch( + new Request("https://cloudhal.dev/mcp", { method: "POST", body: "x" }), + env(), + ); + expect(form.status).toBe(415); + const broken = await worker.fetch( + new Request("https://cloudhal.dev/mcp", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{", + }), + env(), + ); + expect(broken.status).toBe(400); + const limited = await rpc(env({}, true), { jsonrpc: "2.0", id: 1, method: "ping" }); + expect(limited.res.status).toBe(429); + }); +}); + +describe("content negotiation", () => { + const get = (e: Env, p: string, accept?: string) => + worker.fetch(new Request(`https://cloudhal.dev${p}`, accept ? { headers: { accept } } : {}), e); + + it("answers Accept: text/markdown with the page's Markdown", async () => { + const res = await get(env(), "/docs/agents/clients", "text/markdown, text/html;q=0.9"); + expect(res.headers.get("content-type")).toBe("text/markdown; charset=utf-8"); + expect(res.headers.get("vary")).toContain("Accept"); + expect(Number(res.headers.get("x-markdown-tokens"))).toBeGreaterThan(0); + expect(await res.text()).toContain("# Connect an agent"); + expect(await (await get(env(), "/", "text/markdown")).text()).toBe("# Cloudhal\n"); + }); + + it("serves everything else from the assets unchanged", async () => { + const e = env({ "/llms.txt": "# llms" }); + const html = await get(e, "/docs/agents/clients", "text/html"); + expect(await html.text()).toBe("clients"); + expect(html.headers.get("vary")).toContain("Accept"); + expect(await (await get(e, "/llms.txt", "text/markdown")).text()).toBe("# llms"); + const none = await get(e, "/docs/missing", "text/markdown"); + expect(none.status).toBe(404); + expect(e.seen).toEqual([ + "/docs/agents/clients", + "/llms.txt", + "/docs/missing.md", + "/docs/missing", + ]); + }); + + it("reads Accept with q values", () => { + expect(wantsMarkdown("text/markdown")).toBe(true); + expect(wantsMarkdown("text/html, text/markdown;q=0.5")).toBe(true); + expect(wantsMarkdown("text/markdown;q=0")).toBe(false); + expect(wantsMarkdown("text/html,*/*")).toBe(false); + expect(wantsMarkdown(null)).toBe(false); + }); +}); diff --git a/apps/docs/tsconfig.json b/apps/docs/tsconfig.json index 6d54adea..e962feb1 100644 --- a/apps/docs/tsconfig.json +++ b/apps/docs/tsconfig.json @@ -4,5 +4,5 @@ "noEmit": true, "declaration": false }, - "include": ["src/lib", "scripts", "test"] + "include": ["src/lib", "src/worker", "scripts", "test"] } diff --git a/apps/docs/wrangler.jsonc b/apps/docs/wrangler.jsonc index f811d945..44940687 100644 --- a/apps/docs/wrangler.jsonc +++ b/apps/docs/wrangler.jsonc @@ -1,14 +1,22 @@ -// cloudhal.dev (ADR-0037): static assets only, no Worker code. -// Until the Starlight site lands (RFC pending), /docs redirects to the docs on GitHub. +// cloudhal.dev (ADR-0037, ADR-0039): the static site plus a small Worker script in front of it +// (src/worker): the read-only docs MCP at /mcp and `Accept: text/markdown` on pages. Everything +// else is served by the assets binding unchanged. { "$schema": "./node_modules/wrangler/config-schema.json", "name": "cloudhal-dev", + "main": "src/worker/index.ts", "compatibility_date": "2026-10-01", "assets": { "directory": "./dist", + "binding": "ASSETS", + "run_worker_first": true, "not_found_handling": "404-page", "html_handling": "drop-trailing-slash" }, + // Docs MCP: 100 requests a minute per IP (RFC-0037 "Security & policy impact"). + "ratelimits": [ + { "name": "MCP_RATE_LIMITER", "namespace_id": "1039", "simple": { "limit": 100, "period": 60 } } + ], "routes": [{ "pattern": "cloudhal.dev", "custom_domain": true }], "workers_dev": false, "preview_urls": false diff --git a/docs/adr/0039-cloudhal-dev-docs-site.md b/docs/adr/0039-cloudhal-dev-docs-site.md index 015dc570..505231b2 100644 --- a/docs/adr/0039-cloudhal-dev-docs-site.md +++ b/docs/adr/0039-cloudhal-dev-docs-site.md @@ -1,6 +1,7 @@ # ADR-0039: cloudhal.dev: Starlight over the repo's docs, a link contract checked in CI, docs for agents - Status: Accepted - Date: 2026-10-21 (decided by Halil) +- Amendment (2026-10-21, phase 3): the docs stay out of the site MCP server; `cloudhal.dev/mcp` is the only docs MCP. See rule 4. - Related: RFC-0037, ADR-0037, ADR-0011, RFC-0004 - Sources: [Starlight](https://starlight.astro.build/) · [starlight-llms-txt](https://github.com/delucis/starlight-llms-txt) · [llms.txt](https://llmstxt.org/) · [Cloudflare docs for agents](https://developers.cloudflare.com/docs-for-agents/) · [Workers static assets: redirects](https://developers.cloudflare.com/workers/static-assets/redirects/) and [headers](https://developers.cloudflare.com/workers/static-assets/headers/) @@ -11,7 +12,7 @@ ADR-0037 makes `cloudhal.dev` the developer domain and the only copy of the docs 1. **Starlight in `apps/docs`**, reading the monorepo's `docs/` through a content-collection loader. Docs change in the same PR as code. The same `cloudhal-dev` Worker serves it with the schemas and `/fetch`. 2. **Generated reference**: CLI, MCP tools, `cloudhal.json`, schemas and changelog are built from source, never written by hand. 3. **Link contract**: a published URL never 404s (moves get a 301 from `apps/docs/redirects.ts`); `/schemas/v1/*` never moves; user-facing links point at cloudhal.dev, not GitHub paths; errors link to `/e/` pages from one error-code registry in `packages/core-contracts`; CI checks all of it. -4. **Docs for agents**: Markdown per page (`.md` and `/index.md`), `Accept: text/markdown` with `x-markdown-tokens`, `llms.txt` and `llms-full.txt` per site and per section, a read-only unauthenticated MCP at `cloudhal.dev/mcp` (`search_docs`, `read_doc`), and the same two `read` tools in `cloudhal mcp`. +4. **Docs for agents**: Markdown per page (`.md` and `/index.md`), `Accept: text/markdown` with `x-markdown-tokens`, `llms.txt` and `llms-full.txt` per site and per section, and a standalone, read-only, unauthenticated MCP at `cloudhal.dev/mcp` (`search_docs`, `read_doc`), rate limited per IP. The site MCP server (`cloudhal mcp`, remote `/mcp`) gets no docs tools (amendment): it is for agents working on sites (content, design, generated UI, comments, schedules), and its how-to knowledge lives in skills, tool descriptions and server instructions. Docs tools there would add tools that narrow-scoped keys never need, a second knowledge source that can contradict the first, and an outbound dependency on cloudhal.dev. Developer agents read the docs from cloudhal.dev; the site server's instructions only name `llms.txt`. 5. **English only, latest release only** while 0.x; versions under `/docs/v/` from 1.0; languages later under `/docs//`. 6. **All ADRs and RFCs are published**, each page showing its status. 7. **Delivered in four phases** (RFC-0037 §7): the site; generated reference, error pages, CI checks and previews; agent endpoints; new content and link migration. @@ -23,6 +24,7 @@ ADR-0037 makes `cloudhal.dev` the developer domain and the only copy of the docs | Mintlify / GitBook | Fast, polished | Vendor lock-in, cost, not on our Worker | Contradicts open source and self-hosting | | Docusaurus | Versioning built in | A second front-end stack | Starlight is Astro, like the renderer | | GitHub only | No work | No search, navigation or Markdown endpoints | The stopgap, not the site | +| Docs tools in the site MCP (Supabase, Sanity) | One server to connect | Irrelevant tools for narrow keys, two knowledge sources, an outbound call from every deployment | Their servers are developer tools; Cloudhal's is content-operational and policy-governed (amendment) | ## Consequences - **Positive.** One docs source; reference that cannot drift; stable URLs; agents get Markdown, indexes and search. diff --git a/docs/adr/README.md b/docs/adr/README.md index 26ebda4a..cb9e1405 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -55,4 +55,4 @@ One file per decision, numbered, immutable once accepted (write a new ADR to sup | [0036](0036-multi-site-installations.md) | Multi-site installations: one Studio, D1 and R2 manage many isolated sites (per-site roles, export and delete), created empty or as editions, served by hostname through `site_hosts` with per-Worker allow-lists, one theme per site Worker and several site Workers per installation (RFC-0034) | | [0037](0037-domain-strategy.md) | Domain strategy: `cloudhal.com` is marketing, a future hosted control plane and the playground; `cloudhal.dev` is docs, schemas, registry and the open-source home; customer sites never run on either (they deploy into the customer's own Cloudflare account), so no third domain or PSL entry | | [0038](0038-public-playground.md) | Public playground: one SQLite Durable Object per visitor holding store, API and renderer; `@cloudhal/storage/do-sqlite` and SQL/layered/fetcher blob stores; a deny-list guard in the Worker and the API with a route-walk test and a fixed Editor principal; 60-minute TTL with a 2-hour cap; private, uncached session pages (RFC-0036) | -| [0039](0039-cloudhal-dev-docs-site.md) | cloudhal.dev: Starlight in `apps/docs` over the repo's `docs/`, generated reference, a link contract checked in CI (301s, permanent `/schemas/v1`, `/e/` pages from one error-code registry), docs for agents (`.md` per page, `Accept: text/markdown`, `llms.txt`, a read-only docs MCP and `search_docs`/`read_doc` in `cloudhal mcp`); all ADRs and RFCs published (RFC-0037) | +| [0039](0039-cloudhal-dev-docs-site.md) | cloudhal.dev: Starlight in `apps/docs` over the repo's `docs/`, generated reference, a link contract checked in CI (301s, permanent `/schemas/v1`, `/e/` pages from one error-code registry), docs for agents (`.md` per page, `Accept: text/markdown`, `llms.txt`, a standalone read-only docs MCP at `cloudhal.dev/mcp`, none in `cloudhal mcp` (amended)); all ADRs and RFCs published (RFC-0037) | diff --git a/docs/agents/docs-for-agents.md b/docs/agents/docs-for-agents.md index afe7c45f..47a46ff7 100644 --- a/docs/agents/docs-for-agents.md +++ b/docs/agents/docs-for-agents.md @@ -11,7 +11,7 @@ https://cloudhal.dev/docs/start/quickstart → HTML https://cloudhal.dev/docs/start/quickstart.md → Markdown ``` -`/index.md` works too. Each Markdown file starts with frontmatter: `title`, `description`, `url` and `source` (the file in the repository). Links in it are absolute, so it reads correctly anywhere. +`/index.md` works too, and so does asking for the page itself with `Accept: text/markdown` (the answer carries `Vary: Accept` and an `x-markdown-tokens` estimate). Each Markdown file starts with frontmatter: `title`, `description`, `url` and `source` (the file in the repository). Links in it are absolute, so it reads correctly anywhere. ## llms.txt @@ -44,6 +44,52 @@ Docs: https://cloudhal.dev/llms.txt (every page also as Markdown: add .md to its Cloudhal's contracts are JSON Schemas at permanent URLs: [`/schemas/v1/index.json`](/schemas/v1/index.json) lists them. Every node an MCP tool returns follows one of them. -## Searching the docs from the MCP server +## Searching the docs: the docs MCP -Planned ([RFC-0037](/docs/rfcs/0037-cloudhal-dev-docs-site)): `search_docs` and `read_doc` tools in the Cloudhal MCP server, so the agent already connected to your site can look up how to use it, for the version you have installed. Until then, the Markdown pages and `llms.txt` above are the way in. +`https://cloudhal.dev/mcp` is a public MCP server for the docs: read-only, no sign-in, no key. It has two tools: + +- `search_docs` with `query` (and optionally `section`: `start`, `agents`, `reference`, `guides`, `contributing`, `adr` or `rfcs`) returns up to 8 pages with title, URL and a snippet; +- `read_doc` with `url` (a `https://cloudhal.dev/…` URL or a path such as `/docs/agents/clients`) returns that page's Markdown. + +It is **separate from Cloudhal's own MCP server** ([`cloudhal mcp`](/docs/agents/clients), or `/mcp` on a deployment), which works on your sites and has no docs tools. Connect the docs MCP to the agent that writes your theme or integrates Cloudhal; an agent that only edits content does not need it. The answers are public docs text, and the only thing sent to cloudhal.dev is the query. Requests are rate limited per IP address. + +Claude Code: + +```sh +claude mcp add --transport http cloudhal-docs https://cloudhal.dev/mcp +``` + +Codex: + +```sh +codex mcp add cloudhal-docs --url https://cloudhal.dev/mcp +``` + +or in `~/.codex/config.toml`: + +```toml +[mcp_servers.cloudhal-docs] +url = "https://cloudhal.dev/mcp" +``` + +Cursor, in `.cursor/mcp.json`: + +```json +{ + "mcpServers": { + "cloudhal-docs": { "url": "https://cloudhal.dev/mcp" } + } +} +``` + +VS Code (GitHub Copilot), in `.vscode/mcp.json`: + +```json +{ + "servers": { + "cloudhal-docs": { "type": "http", "url": "https://cloudhal.dev/mcp" } + } +} +``` + +Claude Desktop and claude.ai: **Settings › Connectors › Add custom connector** with the URL `https://cloudhal.dev/mcp` and no authentication. Other clients take the same URL as a Streamable HTTP server without headers. diff --git a/docs/rfcs/0037-cloudhal-dev-docs-site.md b/docs/rfcs/0037-cloudhal-dev-docs-site.md index 4d0f9b3f..12b1d569 100644 --- a/docs/rfcs/0037-cloudhal-dev-docs-site.md +++ b/docs/rfcs/0037-cloudhal-dev-docs-site.md @@ -1,6 +1,7 @@ # RFC-0037: cloudhal.dev, the developer site: docs from the repo, a link contract, docs for agents - Status: Accepted (2026-10-21, by Halil; ADR-0039). Open questions decided as recommended: all ADRs and RFCs are published with their status shown, error codes live in one registry in `packages/core-contracts`, and the docs MCP waits for a registry. +- Amendment (2026-10-21, phase 3): no docs tools in `cloudhal mcp`; the standalone `cloudhal.dev/mcp` is the only docs MCP, with a build-time index instead of Pagefind's. See §4 and Implementation notes. - Author: Halil (Madraka), drafted with Claude · Date: 2026-10-21 - Related: - ADR-0037 (domains): `cloudhal.dev` is the developer domain; `cloudhal.dev/docs` is the only copy of the docs; `/schemas/v1/*` is permanent. @@ -9,7 +10,7 @@ - Prior art: Cloudflare's developer docs ([docs for agents](https://developers.cloudflare.com/docs-for-agents/): `/index.md` per page, `Accept: text/markdown`, `/llms.txt` and `/llms-full.txt` per product), Sanity and Strands (`.md` per page, `llms.txt`), Microsoft Learn's docs MCP server, Astro Starlight with `starlight-llms-txt`. ## Summary -cloudhal.dev becomes an Astro Starlight site in `apps/docs`, built from the Markdown in the monorepo's `docs/` folder and deployed as the `cloudhal-dev` Worker that already serves the schemas. Every URL it publishes is a contract: it never breaks, moves get a redirect, and CI checks every link that Cloudhal's code, packages and sites point at. Every page is also published for agents, as Markdown at `.md` and through content negotiation, indexed in `/llms.txt`, and searchable through a read-only MCP endpoint and a `search_docs` tool in `cloudhal mcp`. +cloudhal.dev becomes an Astro Starlight site in `apps/docs`, built from the Markdown in the monorepo's `docs/` folder and deployed as the `cloudhal-dev` Worker that already serves the schemas. Every URL it publishes is a contract: it never breaks, moves get a redirect, and CI checks every link that Cloudhal's code, packages and sites point at. Every page is also published for agents, as Markdown at `.md` and through content negotiation, indexed in `/llms.txt`, and searchable through a read-only MCP endpoint at `cloudhal.dev/mcp`. ## Motivation - **Links already point here.** The 13 npm packages' `homepage`, seven links on cloudhal.com, 89 schema `$id`s, the fetcher's User-Agent and the playground Worker's documented `deployUrl` all name cloudhal.dev URLs. Until #33 none of them resolved. Today `/docs/*` lands on GitHub's file view, which is fine for contributors but poor for users: no navigation, no search, no Markdown for agents. @@ -72,14 +73,14 @@ Who links where: - **Content negotiation.** A small Worker script in front of the assets answers `Accept: text/markdown` on any page with its Markdown, sets `Vary: Accept`, and adds `x-markdown-tokens` (an estimate), as Cloudflare does. Everything else is served by the assets binding unchanged. - **Indexes.** `/llms.txt` lists every page with its one-line description, grouped by section. `/llms-full.txt` is the whole docs in one file. Each section has its own `llms.txt` and `llms-full.txt`. - **Docs MCP.** `https://cloudhal.dev/mcp` is a stateless, read-only MCP endpoint with no authentication. It has two tools: - - `search_docs(query, section?)` returns up to 8 hits with title, URL and snippet, from the Pagefind index or a build-time FTS index; + - `search_docs(query, section?)` returns up to 8 hits with title, URL and snippet, from a build-time index; - `read_doc(url)` returns one page's Markdown. - `cloudhal mcp` gains the same two tools, so the agent already connected to a site can look up how to use it. They proxy to cloudhal.dev and are read-only, so policy class `read`. -- **AGENTS.md for users.** `/docs/start/agents` explains how to point an agent at Cloudhal: the MCP server, the docs MCP, the `llms.txt`. The page is written for agents to read. + *Amended 2026-10-21:* `cloudhal mcp` does **not** gain these tools. The site MCP server is for agents working on sites (content, design, generated UI, comments, schedules), and its how-to knowledge lives in skills, tool descriptions and server instructions. Docs tools there would add tools that narrow-scoped keys never need, a second knowledge source that can contradict the first, and an outbound dependency on cloudhal.dev. Microsoft Learn, AWS, Mintlify and GitBook ship docs as a standalone, unauthenticated MCP server; Supabase and Sanity put docs tools in their product server, but those servers are developer tools, while Cloudhal's is content-operational and policy-governed. Developer agents connect `cloudhal.dev/mcp` (or read `llms.txt`); the site server's instructions name `llms.txt` and nothing more. +- **AGENTS.md for users.** `/docs/agents/docs-for-agents` explains how to point an agent at Cloudhal: the MCP server, the docs MCP, the `llms.txt`. The page is written for agents to read. ### 5. Search, versions, languages -- Search: Pagefind (Starlight's default; static, no service). The docs MCP reuses its index. +- Search: Pagefind (Starlight's default; static, no service) for people. The docs MCP has its own index, built from the same page list (Implementation notes). - Versions: while Cloudhal is 0.x, only the latest release is documented, with an "applies to 0.3.x" note in the header taken from the packages' version. At 1.0, older majors move to `/docs/v1/` and so on. - Languages: English only (the default language rule). Starlight's i18n and the Weblate setup (`docs/guides/translations-weblate.md`) can add languages later without URL changes, under `/docs//`. @@ -92,7 +93,7 @@ Who links where: ### 7. Phases 1. Starlight in `apps/docs` over the existing `docs/`, with the IA above, Pagefind, `.md` per page, `llms.txt`/`llms-full.txt`, and the redirects that replace today's GitHub 302s. 2. Generated reference (CLI, MCP tools, config, schemas, changelog), `/e/` pages, link checks in CI, deploy from CI with PR previews. -3. Content negotiation Worker, `https://cloudhal.dev/mcp`, `search_docs`/`read_doc` in `cloudhal mcp`. +3. Content negotiation Worker and `https://cloudhal.dev/mcp` (no docs tools in `cloudhal mcp`, amended). 4. New content: `docs/start/*` and `docs/concepts/*`. Then point the CLI, Studio, cloudhal.com and the template at the new URLs. ## Alternatives considered @@ -112,7 +113,7 @@ Who links where: ## Security & policy impact - The docs MCP endpoint is read-only, unauthenticated and stateless. It is rate limited per IP with a Workers rate-limiting binding. It reads only the built docs, never a site. -- `search_docs` and `read_doc` in `cloudhal mcp` are `read` tools. Their output is public docs text; they never send site data to cloudhal.dev, only the query. +- The site MCP server sends nothing to cloudhal.dev (amended: no docs tools there). - The CI deploy token can edit only the `cloudhal-dev` Worker. - No cookies, no analytics beyond Cloudflare's aggregate Web Analytics, no third-party scripts. @@ -123,3 +124,4 @@ Who links where: ## Implementation notes - **Phase 1** (`apps/docs`): a custom content loader (`src/lib/repo-docs.ts`) reads `docs/` and the site's own pages (`src/lib/site-pages.ts`). The H1 becomes the title; the description is the first text under `## Summary` or `## Context`; relative links become site paths, or GitHub links for files outside `docs/`. Folders without a README get a generated index (`/docs/rfcs`). `scripts/postbuild.ts` writes the schemas, the `.md` copies, `llms.txt`/`llms-full.txt` (site and per section) and `_redirects` from `src/lib/redirects.ts` from the same page list. The `starlight-llms-txt` plugin is not used: one list feeds the HTML, the Markdown and the indexes, so they cannot disagree. +- **Phase 3** (`apps/docs/src/worker`, amended 2026-10-21): the `cloudhal-dev` Worker now has a script, run before the assets (`run_worker_first`). `/mcp` is the docs MCP: Streamable HTTP, stateless (no `Mcp-Session-Id`, no event stream; GET and DELETE answer 405), one JSON answer per POST, protocol versions 2024-11-05 to 2025-11-25, tools `search_docs` and `read_doc` (read-only annotations), 100 requests a minute per IP through the `MCP_RATE_LIMITER` binding. `scripts/postbuild.ts` writes the search index (`/_mcp/search-index.json`: plain text and an inverted index with title, description and path boosts, from the same page list as the HTML) and the Worker ranks with BM25 plus prefix matches. Pagefind's index is built for the browser (fragments fetched on demand), so it was not reused. `read_doc` takes only cloudhal.dev URLs or site paths and answers the page's `.md` copy. `Accept: text/markdown` on a page answers that copy with `Vary: Accept` and `x-markdown-tokens` (characters / 4); every other request goes to the assets binding unchanged, and page HTML gains `Vary: Accept`. diff --git a/packages/mcp-server/src/server-instructions.ts b/packages/mcp-server/src/server-instructions.ts index d1953203..d4e7e350 100644 --- a/packages/mcp-server/src/server-instructions.ts +++ b/packages/mcp-server/src/server-instructions.ts @@ -21,7 +21,9 @@ Languages: a site lists its locales (get_site). To translate a page or entry, ca Schedules: to publish later, propose or schedule a release with the entries (propose_release, schedule_release); a person approves the pinned versions in Studio and Cloudhal publishes them at that time — do not poll and do not call again, get_release shows the outcome. Scheduled content is locked; ask a person to unschedule it. Embargoes and expiries are times on the entry (set_embargo, schedule_unpublish), enforced by the site itself; lifting an embargo early is always a person's decision (lift_embargo). Inside an automation run (begin_run) you have a budget and an end time; once you read reader text, 404 paths, analytics referrers or fetched pages, the run can only read, label, hide or pause. Close it with end_run. -If you are following a Cloudhal skill (read it from the skill:// resources), send its name in the request \`_meta["cloudhal.skill"]\`; your calls are then limited to the scopes that skill declared.`; +If you are following a Cloudhal skill (read it from the skill:// resources), send its name in the request \`_meta["cloudhal.skill"]\`; your calls are then limited to the scopes that skill declared. + +Cloudhal's developer docs (themes, the CLI, integrations; not needed to work on site content) are at https://cloudhal.dev/llms.txt, every page also as Markdown.`; const MODE_TEXT: Record = { plan: "You are in PLAN mode: read and preview only. Do not try to change anything; describe the changes you would make so a person can decide.",