diff --git a/.changeset/reference-consumers.md b/.changeset/reference-consumers.md new file mode 100644 index 00000000..20235544 --- /dev/null +++ b/.changeset/reference-consumers.md @@ -0,0 +1,17 @@ +--- +"@cloudhal/core-ts": minor +"@cloudhal/api": minor +"@cloudhal/cli": patch +"@cloudhal/cloudflare": patch +"@cloudhal/mcp-server": patch +--- + +The consumers of reference fields (RFC-0045 §2, rollout step 2b, #114). Both new field types stay refused on every write path in production, so none of this is reachable before rollout step 7. The shared suites run it through `SiteGraphStoreOptions.fieldTypes`. + +- **Assets inside repeater items.** `fieldAssetRefs` walks items and records `fields..` with each use's item path. `fieldAssetIssues` checks sub-field types (a PDF in an image sub-field is refused) and reports at every item that uses the file. Skips match issue paths by prefix, in the store and in the MCP precheck alike, so top-level behaviour is unchanged. +- **Field-operation rewrites** write their entries' `entry_refs` rows. A rewrite that changes a reference value is guarded on its target, and every rewrite in a site with reference fields is guarded on the collection's definitions. +- **Releases** no longer treat reference values as dependencies, so a referenced entry is neither locked nor required live. +- **Language editions** (`forEdition`) keep only what the edition's own write accepts. A reference needs a member of its group in an allowed collection that meets the field's `where`. A single-target field keeps one id, and a multiple field keeps at most `maxItems`. An entry holds at most 100 references, and a repeater at most `maxItems` items. Values are read through a running rename's old key. Everything not carried, including repeater items left empty, is reported with its reason through `onDropped`. A re-save keeps the values the edition entry already holds, as its own write would. +- **CLI.** `cloudhal import --dry-run` warns about references that will show nothing, and about originals relinked into another group. +- **Cloudflare.** `siteFieldTypeSupport` reads the site Worker's `fields` feature. `FIELDS_FEATURE_VERSION` and `FIELDS_SUPPORTED` are exported from `@cloudhal/core-ts/fields`. Nothing passes them to a store before step 7. +- **API.** `GET /sites/:siteId/entries/pick` is the entry picker. Its search folds case and diacritics on both sides with `foldName`, Turkish İ and ı included, and reads at most 1,000 entries per request. When that runs out before 20 matches, it answers `truncated: true` with a `cursor` to continue from. `GET /sites/:siteId/entries/:entryId/references` lists an entry's referrers, each with the language versions that show it. Both need `content:read`. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index e0893f3c..b941ed5b 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -642,7 +642,11 @@ All new UI follows `kumo-design`, `studio-ui` and DESIGN.md "Fields", imports on - **repeater:** a nested sub-field list (rows like `FieldRow`, with up/down and ⋯), the maximum and the item label; - the ⋯ menu's operations gain the sub-field forms and the new `change_field_type` inputs (§7). - **API** (site-scoped, under the `/sites/:siteId` membership check): - - `GET /sites/:siteId/entries/pick?collections=a,b&q=&locale=&limit=` answers at most 20 `{groupId, id, title, collection, status, locale}` per page, never archived ones. It needs `content:read`. `q` is at most 100 characters, matched as a case-folded substring on titles and slugs (`includes`, no regex: rule 13), under a per-member window of the existing `rate_limits` counters (`packages/api/src/rate-limits.ts`, migration 0030). + - `GET /sites/:siteId/entries/pick?collections=a,b&q=&locale=&limit=&cursor=` (amended in 2b, PR #206). It needs `content:read`. + - The answer is `{entries, cursor?, truncated?}`: at most 20 `{groupId, id, title, collection, status, locale}`, never archived ones, in key order (collection, locale, slug). + - Each allowed collection (and locale) is a key range of `nodes_site_kind_key`, read with a keyset cursor. A request reads at most 1,000 rows. When the budget ends before `limit` matches, the answer says `truncated: true` and carries the `cursor` to continue from, so any match can be found. + - `q` is at most 100 characters. Both it and each row's title and slug are folded with `foldName` (case, diacritics, Turkish İ/ı) and matched as a substring with no pattern (rule 13). A `q` that folds to nothing matches nothing. + - The existing `rate_limits` counters (`packages/api/src/rate-limits.ts`, migration 0030) give a per-member window. - `GET /sites/:siteId/entries/:id/references` answers the referrers from `entry_refs`, expanded to members, paged. - **Accessibility:** - Each repeater item is a `fieldset` whose `legend` is its label ("Speaker 2: Ada Lovelace"), an ICU `selectordinal`/`plural` message, never concatenated. diff --git a/packages/api/src/entry-pick.ts b/packages/api/src/entry-pick.ts new file mode 100644 index 00000000..b36256e4 --- /dev/null +++ b/packages/api/src/entry-pick.ts @@ -0,0 +1,116 @@ +/** + * RFC-0045 §5: the entry picker's search, bounded per request and complete across requests. + * + * - **Where it reads.** An entry's node key is `::` (`NODE_SPEC`), and + * `nodes_site_kind_key (site_id, kind, key)` is a unique index. So each allowed collection, or + * collection and locale, is one index range, read in key order with a keyset cursor, and every + * row read is a row returned: the `WHERE` names only indexed columns. + * - **The budget.** At most `PICK_ROWS` rows per request, across collections. When the budget + * ends before `limit` matches, the answer says `truncated: true` and carries a `cursor`. Asking + * again with it continues where this request stopped, so any matching entry can be found. + * - **The match.** `q` and each row's title and slug are folded with the same `foldName` (case, + * diacritics, Turkish İ/ı), and matched as a substring, with no pattern over the input (rule 13). + * A `q` with nothing left once folded (only punctuation) matches nothing. + * + * Archived entries are skipped (they count toward the budget). Results come in key order: + * collection, locale, slug. + */ +import type { ContentEntry } from "@cloudhal/core-contracts"; +import { foldName } from "@cloudhal/core-ts/names"; +import type { SqlDriver } from "@cloudhal/storage"; + +/** Rows one picker request reads at most. */ +export const PICK_ROWS = 1000; + +export interface PickQuery { + q: string; + collections: readonly string[]; + locale?: string | undefined; + limit: number; + /** The `cursor` of an earlier answer: the last node key it read. */ + cursor?: string | undefined; + /** Rows to read at most (default `PICK_ROWS`; tests pass less). */ + budget?: number | undefined; +} + +export interface PickCard { + groupId: string; + id: string; + title: string; + collection: string; + status: ContentEntry["status"]; + locale: string; +} + +export interface PickPage { + entries: PickCard[]; + /** Where to continue; absent once every allowed collection was read to its end. */ + cursor?: string; + /** The row budget ended before `limit` matches were found: ask again with `cursor`. */ + truncated?: true; +} + +interface Row extends Record { + key: string; + id: string; + title: string | null; + status: ContentEntry["status"]; + grp: string | null; +} + +/** The SQL of one collection range (exported for the query-plan test). */ +export const PICK_RANGE_SQL = `SELECT key, id, json_extract(body, '$.title') AS title, + json_extract(body, '$.status') AS status, json_extract(body, '$.translation.group') AS grp + FROM nodes WHERE site_id = ? AND kind = 'contentEntry' AND key >= ? AND key > ? AND key < ? + ORDER BY key LIMIT ?`; + +/** The key range of a collection (and locale): `` up to the next character after `:`. */ +function rangeOf(collection: string, locale: string | undefined) { + const prefix = locale === undefined ? `${collection}:` : `${collection}:${locale}:`; + return { prefix, upper: `${prefix.slice(0, -1)};` }; +} + +/** One picker answer. */ +export async function pickEntries( + sql: SqlDriver, + siteId: string, + query: PickQuery, +): Promise { + const raw = query.q.trim(); + const q = foldName(raw); + if (raw !== "" && q === "") return { entries: [] }; + const hits = (text: string | null) => text !== null && foldName(text).includes(q); + const budget = query.budget ?? PICK_ROWS; + const entries: PickCard[] = []; + let read = 0; + let last = query.cursor; + // Ranges in the order of their keys, not of the names: "post-x:" and "news2:" sort before + // "post:" and "news:" ('-' and digits sort below ':'), and the cursor walks key order. + const ranges = [...new Set(query.collections)] + .map((collection) => ({ collection, ...rangeOf(collection, query.locale) })) + .sort((a, b) => (a.prefix < b.prefix ? -1 : a.prefix > b.prefix ? 1 : 0)); + for (const { collection, prefix, upper } of ranges) { + if (last !== undefined && last >= upper) continue; + // One read per range, for the rest of the budget: the range ends first, or the budget does. + const want = budget - read; + const rows = await sql.all(PICK_RANGE_SQL, [siteId, prefix, last ?? "", upper, want]); + for (const r of rows) { + read++; + last = r.key; + if (r.status === "archived") continue; + const cut = r.key.indexOf(":", collection.length + 1); + if (q !== "" && !hits(r.title) && !hits(r.key.slice(cut + 1))) continue; + entries.push({ + groupId: r.grp ?? r.id, + id: r.id, + title: r.title ?? "", + collection, + status: r.status, + locale: r.key.slice(collection.length + 1, cut), + }); + if (entries.length >= query.limit) return { entries, cursor: r.key }; + } + if (read >= budget) return { entries, ...(last ? { cursor: last } : {}), truncated: true }; + } + return { entries }; +} diff --git a/packages/api/src/playground.ts b/packages/api/src/playground.ts index 499aa741..a10d84f7 100644 --- a/packages/api/src/playground.ts +++ b/packages/api/src/playground.ts @@ -144,6 +144,9 @@ export const PLAYGROUND_ALLOW: ReadonlySet = new Set([ `POST ${S}/entries/:entryId/preview`, `GET ${S}/entries/:entryId/structured-data`, `POST ${S}/entries/preview-fragment`, + // RFC-0045 §5: the entry picker and an entry's referrers, read-only (content:read). + `GET ${S}/entries/pick`, + `GET ${S}/entries/:entryId/references`, `GET ${S}/entries/:entryId`, `PUT ${S}/entries/:entryId`, `DELETE ${S}/entries/:entryId`, diff --git a/packages/api/src/routes/entries.ts b/packages/api/src/routes/entries.ts index b3a0ad86..f4bd1052 100644 --- a/packages/api/src/routes/entries.ts +++ b/packages/api/src/routes/entries.ts @@ -15,6 +15,7 @@ import { editsLiveCorrections, entryLimit, personOf } from "../own.ts"; import { param } from "../params.ts"; import { type DraftPreview, previewResponse } from "../preview.ts"; import { previewStructuredData } from "../structured-data.ts"; +import { entryReferences } from "./entry-references.ts"; const STATUSES = new Set(["draft", "review", "published", "archived"]); @@ -56,6 +57,8 @@ export const entries: Hono = new Hono() // `filter` is the RFC-0002 §5 language; a FilterError maps to 400 with suggestions. return c.json(await c.get("store").listContentEntries(param(c, "siteId"), base)); }) + // RFC-0045 §5: the entry picker and an entry's referrers (before `/:entryId`). + .route("/", entryReferences) .post("/", async (c) => { const siteId = param(c, "siteId"); if (!(await c.get("store").getSite(siteId))) throw notFound("site not found"); diff --git a/packages/api/src/routes/entry-references.ts b/packages/api/src/routes/entry-references.ts new file mode 100644 index 00000000..43afe855 --- /dev/null +++ b/packages/api/src/routes/entry-references.ts @@ -0,0 +1,158 @@ +/** + * RFC-0045 §5 "API": the entry picker and an entry's referrers, site-scoped under the + * `/sites/:siteId` membership check, both `content:read`. + * + * - `GET …/entries/pick?collections=a,b&q=&locale=&limit=&cursor=`: at most 20 `{groupId, id, + * title, collection, status, locale}` of the allowed collections, never archived ones, with a + * `cursor` and `truncated` when the row budget ended first (`entry-pick.ts`). `q` is at most + * 100 characters and refused before it is read further; a per-member window of the + * `rate_limits` counters applies. + * - `GET …/entries/:entryId/references?limit=&offset=`: the referrers of the entry's translation + * group from `entry_refs`, published holders first, each holder with the same-site members that + * show its value (`shared` 1, or 2 without their own value), paged. + */ +import type { ContentEntry } from "@cloudhal/core-contracts"; +import { memberGroup } from "@cloudhal/core-ts/fields"; +import type { SqlDriver } from "@cloudhal/storage"; +import type { Context } from "hono"; +import { Hono } from "hono"; +import type { Env } from "../context.ts"; +import { pickEntries } from "../entry-pick.ts"; +import { apiError, badRequest, notFound } from "../errors.ts"; +import { param } from "../params.ts"; +import { countHit } from "../rate-limits.ts"; + +const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const LOCALE = /^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8}){0,3}$/; +const MAX_Q = 100; +/** A node key: collection, locale and slug (each well under 100 characters). */ +const MAX_CURSOR = 400; +const PICK_LIMIT = 20; +/** Picker searches per member and site per minute. */ +const PICK_WINDOW = { hits: 120, ms: 60_000 }; +const REFERRERS_LIMIT = 100; + +function intParam(raw: string | undefined, max: number, fallback: number): number { + if (raw === undefined || raw === "") return fallback; + const n = Number(raw); + if (!Number.isInteger(n) || n < 0) throw badRequest("limit and offset are non-negative integers"); + return Math.min(n, max); +} + +/** The picker's query, checked before any read. */ +function pickQuery(c: Context) { + const q = c.req.query("q") ?? ""; + if (q.length > MAX_Q) throw badRequest(`q is at most ${MAX_Q} characters`); + const collections = (c.req.query("collections") ?? "").split(",").filter((s) => s !== ""); + if (collections.length === 0 || collections.length > 5 || !collections.every((s) => SLUG.test(s))) + throw badRequest("collections lists one to five collection keys"); + const locale = c.req.query("locale") || undefined; + if (locale !== undefined && !LOCALE.test(locale)) throw badRequest("locale is a language tag"); + const limit = Math.max(1, intParam(c.req.query("limit"), PICK_LIMIT, PICK_LIMIT)); + const cursor = c.req.query("cursor") || undefined; + if (cursor !== undefined && cursor.length > MAX_CURSOR) + throw badRequest("cursor is not one this route gave"); + return { q, collections, locale, limit, cursor }; +} + +async function rateLimited(c: Context, siteId: string): Promise { + const who = c.get("actor")?.id ?? "anonymous"; + const count = await countHit(c.get("store").sql, { + key: `entries-pick:${siteId}:${who}`, + windowMs: PICK_WINDOW.ms, + now: c.get("now")().getTime(), + }); + return count > PICK_WINDOW.hits; +} + +interface Card { + groupId: string; + id: string; + title: string; + collection: string; + status: ContentEntry["status"]; + locale: string; +} + +const cardOf = (e: ContentEntry): Card => ({ + groupId: memberGroup(e), + id: e.id, + title: e.title, + collection: e.collection, + status: e.status, + locale: e.locale, +}); + +interface RefRow { + holder: string; + field: string; + shared: number; + root_key: string; + source_status: string; + body: string; +} + +/** One page of the group's rows, each holder with the members that show its value. */ +async function referrers( + sql: SqlDriver, + siteId: string, + page: { group: string; limit: number; offset: number }, +) { + const rows = await sql.all>( + `SELECT r.source_id AS holder, r.field AS field, r.shared AS shared, r.root_key AS root_key, + r.source_status AS source_status, n.body AS body + FROM entry_refs r JOIN nodes n ON n.id = r.source_id AND n.site_id = r.site_id + WHERE r.site_id = ? AND r.target_id = ? + ORDER BY r.source_status = 'published' DESC, r.source_id, r.field LIMIT ? OFFSET ?`, + [siteId, page.group, page.limit + 1, page.offset], + ); + const shown = rows.slice(0, page.limit); + const holders = [...new Set(shown.map((r) => r.holder))]; + const members = holders.length + ? await sql.all<{ body: string }>( + `SELECT body FROM nodes WHERE site_id = ? AND kind = 'contentEntry' + AND json_extract(body, '$.translation.group') IN (SELECT value FROM json_each(?)) + AND json_extract(body, '$.translation.source.siteId') IS NULL`, + [siteId, JSON.stringify(holders)], + ) + : []; + const inheriting = members.map((m) => JSON.parse(m.body) as ContentEntry); + const items = shown.map((r) => ({ + ...cardOf(JSON.parse(r.body) as ContentEntry), + field: r.field, + shownBy: inheriting + .filter( + (m) => + m.translation?.group === r.holder && + // Only the row's own holder is left out: another holder on this page may still show it. + m.id !== r.holder && + (r.shared === 1 || !Object.hasOwn(m.fields ?? {}, r.root_key)), + ) + .map(cardOf), + })); + return { + referrers: items, + ...(rows.length > page.limit ? { next: page.offset + page.limit } : {}), + }; +} + +export const entryReferences: Hono = new Hono() + .get("/pick", async (c) => { + const siteId = param(c, "siteId"); + const query = pickQuery(c); + if (!(await c.get("store").getSite(siteId))) throw notFound("site not found"); + if (await rateLimited(c, siteId)) + throw apiError(429, "rate_limited", "too many picker searches; wait a minute"); + return c.json(await pickEntries(c.get("store").sql, siteId, query)); + }) + .get("/:entryId/references", async (c) => { + const store = c.get("store"); + const entry = await store.getContentEntry(param(c, "entryId")); + if (!entry || entry.siteId !== param(c, "siteId")) throw notFound("entry not found"); + const page = { + group: memberGroup(entry), + limit: Math.max(1, intParam(c.req.query("limit"), REFERRERS_LIMIT, 20)), + offset: intParam(c.req.query("offset"), Number.MAX_SAFE_INTEGER, 0), + }; + return c.json({ groupId: page.group, ...(await referrers(store.sql, entry.siteId, page)) }); + }); diff --git a/packages/api/src/scopes.ts b/packages/api/src/scopes.ts index b851f186..be6f6190 100644 --- a/packages/api/src/scopes.ts +++ b/packages/api/src/scopes.ts @@ -172,6 +172,9 @@ export const ROUTE_SCOPES: readonly (readonly [Method, string, RouteRequirement] ["POST", `${S}/entries/:entryId/preview`, "content:read"], // the body editor's inline preview of one ```surface (writes nothing) ["POST", `${S}/entries/preview-fragment`, "content:read"], + // RFC-0045 §5: the entry picker and an entry's referrers + ["GET", `${S}/entries/pick`, "content:read"], + ["GET", `${S}/entries/:entryId/references`, "content:read"], ["GET", `${S}/entries/:entryId`, "content:read"], ["PUT", `${S}/entries/:entryId`, "content:draft:own"], ["DELETE", `${S}/entries/:entryId`, "content:draft:own"], diff --git a/packages/api/src/testing-references.ts b/packages/api/src/testing-references.ts new file mode 100644 index 00000000..7b5044b0 --- /dev/null +++ b/packages/api/src/testing-references.ts @@ -0,0 +1,213 @@ +/** + * Shared API suite for RFC-0045 rollout step 2: the entry picker, an entry's referrers, and the + * status codes of the reference refusals on the entry routes. The types stay refused in + * production; the suite wraps the factory's storage in a store with `fieldTypes`. + */ +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import { nodeHash, SiteGraphStore } from "@cloudhal/core-ts"; +import { afterAll, beforeEach, describe, expect, it } from "vitest"; +import { createApi } from "./app.ts"; +import type { ApiStoreFactory } from "./testing.ts"; + +const human = { kind: "human" as const, id: "user_apitest00001", name: "Tester" }; +const L = (label: string) => ({ en: { label } }); +type Json = Record; + +export function describeReferencesApi(label: string, factory: ApiStoreFactory): void { + describe(`API references (RFC-0045 step 2): ${label}`, () => { + let store: SiteGraphStore; + let site: Site; + let app: ReturnType; + const headers = { "content-type": "application/json", "x-test-user": "1" }; + const call = async (path: string, init: RequestInit = {}) => { + const res = await app.request(path, { headers, ...init }); + return { status: res.status, body: (await res.json().catch(() => ({}))) as Json }; + }; + const entry = (collection: string, over: Partial = {}) => + store.createContentEntry({ + siteId: site.id, + collection, + slug: `${collection}-${Math.random().toString(36).slice(2, 8)}`, + locale: "en", + status: "draft", + title: "Untitled", + body: { format: "markdown", value: "Body" }, + fields: {}, + tags: [], + author: human, + ...over, + } as never); + + beforeEach(async () => { + const base = await factory.create(); + store = new SiteGraphStore({ + sql: base.sql, + blobs: base.blobs, + fieldTypes: { supported: () => true }, + }); + const project = await store.createProject({ + name: "Refs", + slug: `refs-${Math.random().toString(36).slice(2, 8)}`, + ownerId: human.id, + siteIds: [], + }); + site = await store.createSite({ + projectId: project.id, + name: "Refs", + slug: `refs-${Math.random().toString(36).slice(2, 8)}`, + defaultLocale: "en", + locales: ["en", "tr"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: { + places: {}, + events: { + fields: [ + { + key: "venue", + type: "reference", + labels: L("Venue"), + localized: false, + collections: ["places"], + }, + { + key: "altVenue", + type: "reference", + labels: L("Alternative venue"), + localized: true, + collections: ["places"], + }, + ], + }, + }, + }); + app = createApi({ + store, + resolveActor: async (c) => (c.req.header("x-test-user") ? human : undefined), + }); + }); + + afterAll(async () => { + await factory.dispose?.(); + }); + + it("picks entries of the allowed collections by a substring, never archived ones", async () => { + const main = await entry("places", { title: "Main Hall", slug: "main-hall" }); + await entry("places", { title: "Old hall", status: "archived" }); + await entry("places", { title: "Garden" }); + await entry("events", { title: "Hall party" }); + const picked = await call(`/api/sites/${site.id}/entries/pick?collections=places&q=HALL`); + expect(picked.status, JSON.stringify(picked.body)).toBe(200); + expect(picked.body.entries).toEqual([ + { + groupId: main.id, + id: main.id, + title: "Main Hall", + collection: "places", + status: "draft", + locale: "en", + }, + ]); + const long = await call( + `/api/sites/${site.id}/entries/pick?collections=places&q=${"a".repeat(100_001)}`, + ); + expect(long.status).toBe(400); + expect((await call(`/api/sites/${site.id}/entries/pick`)).status).toBe(400); + expect( + (await call("/api/sites/site_01HZX8K3QNONE/entries/pick?collections=places")).status, + ).toBe(404); + }); + + it("folds case and diacritics on both sides, Turkish İ and ı included", async () => { + const cicek = await entry("places", { title: "Çiçek Pasajı", slug: "cicek-pasaji" }); + const kongre = await entry("places", { title: "İstanbul Kongre", slug: "kongre" }); + const pick = async (q: string) => + ( + ( + await call( + `/api/sites/${site.id}/entries/pick?collections=places&q=${encodeURIComponent(q)}`, + ) + ).body.entries as { id: string }[] + ).map((e) => e.id); + expect(await pick("ÇİÇEK")).toEqual([cicek.id]); + expect(await pick("Çiçek")).toEqual([cicek.id]); + expect(await pick("pasaji")).toEqual([cicek.id]); + expect(await pick("istanbul")).toEqual([kongre.id]); + expect(await pick("İSTANBUL")).toEqual([kongre.id]); + expect(await pick("KONG")).toEqual([kongre.id]); + }); + + it("shows a translation under the original's non-localized row even when it holds a row itself", async () => { + const hall = await entry("places", { title: "Main Hall" }); + const original = await entry("events", { + title: "Launch", + status: "published", + fields: { venue: hall.id }, + }); + const translation = await entry("events", { + title: "Lansman", + locale: "tr", + slug: `${original.slug}-tr`, + status: "published", + fields: { altVenue: hall.id }, + translation: { + group: original.id, + method: "human", + source: { + id: original.id, + locale: original.locale, + rev: original.rev, + hash: await nodeHash(original), + }, + }, + }); + // every page, for a page size: the result must not depend on where the pages are cut + const rows = async (limit: number) => { + const out: { id: string; field: string; shownBy: { id: string }[] }[] = []; + for (let offset: number | undefined = 0; offset !== undefined; ) { + const page = ( + await call( + `/api/sites/${site.id}/entries/${hall.id}/references?limit=${limit}&offset=${offset}`, + ) + ).body as { referrers: typeof out; next?: number }; + out.push(...page.referrers); + offset = page.next; + } + return out; + }; + for (const limit of [20, 1]) { + const venue = (await rows(limit)).find((r) => r.field === "venue"); + expect(venue?.id).toBe(original.id); + expect(venue?.shownBy.map((m) => m.id)).toEqual([translation.id]); + } + const alt = (await rows(20)).find((r) => r.field === "altVenue"); + expect(alt).toMatchObject({ id: translation.id, shownBy: [] }); + }); + + it("lists referrers with the members that show their value, and refuses what strands them", async () => { + const hall = await entry("places", { title: "Main Hall" }); + const event = await entry("events", { + title: "Launch", + status: "published", + fields: { venue: hall.id }, + }); + const refs = await call(`/api/sites/${site.id}/entries/${hall.id}/references`); + expect(refs.status, JSON.stringify(refs.body)).toBe(200); + expect(refs.body.groupId).toBe(hall.id); + const [first] = refs.body.referrers as { id: string; field: string; shownBy: unknown[] }[]; + expect(first).toMatchObject({ id: event.id, field: "venue", shownBy: [] }); + const del = await call(`/api/sites/${site.id}/entries/${hall.id}?rev=${hall.rev}`, { + method: "DELETE", + }); + expect(del.status).toBe(409); + expect(del.body.error).toBe("entry_referenced"); + const bad = await call(`/api/sites/${site.id}/entries/${event.id}`, { + method: "PUT", + body: JSON.stringify({ ...event, fields: { venue: "entry_01HZX8K3QGONE" } }), + }); + expect(bad.status).toBe(422); + expect(bad.body.error).toBe("unknown_entry"); + }); + }); +} diff --git a/packages/api/test/entry-pick.test.ts b/packages/api/test/entry-pick.test.ts new file mode 100644 index 00000000..07781d35 --- /dev/null +++ b/packages/api/test/entry-pick.test.ts @@ -0,0 +1,157 @@ +/** + * RFC-0045 §5: the picker's bounded search (`pickEntries`) on every driver: a keyset over + * `nodes_site_kind_key`, `foldName` on both sides, `truncated` and a `cursor` when the row budget + * ends first, so any matching entry is found by asking again. Node rows are inserted directly. + */ +import { migrate, migrations, type SqlDriver } from "@cloudhal/storage"; +import { D1Driver } from "@cloudhal/storage/d1"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { createMiniflareD1, createMiniflareDoSqlite } from "@cloudhal/storage/testing"; +import { afterAll, describe, expect, it } from "vitest"; +import { PICK_RANGE_SQL, type PickQuery, pickEntries } from "../src/entry-pick.ts"; + +const SITE = "site_01HZX8K3QPICK"; + +let d1: Awaited> | undefined; +let dosql: Awaited> | undefined; +afterAll(async () => { + await d1?.dispose(); + await dosql?.dispose(); +}); + +const drivers: [string, () => Promise][] = [ + ["better-sqlite3", () => BetterSqlite3Driver.open(":memory:")], + [ + "Cloudflare D1 (Miniflare)", + async () => { + d1 ??= await createMiniflareD1(); + return new D1Driver(d1.db); + }, + ], + [ + "Durable Object SQLite (Miniflare)", + async () => { + dosql ??= await createMiniflareDoSqlite(); + return dosql.driver(`pick-${Math.random().toString(36).slice(2)}`); + }, + ], +]; + +let n = 0; +async function world(open: () => Promise) { + const sql = await open(); + await migrate(sql, migrations); + await sql.run("DELETE FROM nodes WHERE site_id = ?", [SITE]); + const add = async (collection: string, slug: string, title: string, over: object = {}) => { + n++; + const id = `entry_PICK${String(n).padStart(8, "0")}`; + const body = { id, collection, locale: "tr", slug, title, status: "draft", ...over }; + await sql.run( + "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, 'contentEntry', ?, 0, ?, ?, 't', 't')", + [id, SITE, `${collection}:tr:${slug}`, `h${n}`, JSON.stringify(body)], + ); + return id; + }; + return { sql, add }; +} + +describe.each(drivers)("pickEntries: %s", (_, open) => { + it("finds an entry past the row budget by following the cursor", async () => { + const { sql, add } = await world(open); + for (let i = 0; i < 12; i++) + await add("places", `a-${String(i).padStart(2, "0")}`, `Room ${i}`); + const late = await add("places", "z-cicek", "Çiçek Pasajı"); + await add("places", "z-old", "Çiçek eski", { status: "archived" }); + await add("people", "b-cicek", "Çiçek Hanım"); + const query: PickQuery = { q: "ÇİÇEK", collections: ["places"], limit: 20, budget: 5 }; + const found: string[] = []; + let pages = 0; + for (let cursor: string | undefined; ; ) { + const page = await pickEntries(sql, SITE, { ...query, cursor }); + pages++; + found.push(...page.entries.map((e) => e.id)); + if (!page.cursor) break; + expect(page.truncated).toBe(true); + cursor = page.cursor; + } + expect(found).toEqual([late]); + expect(pages).toBe(3); + }); + + it("stops at limit with a cursor, reads across collections, and matches nothing for punctuation", async () => { + const { sql, add } = await world(open); + const a = await add("halls", "one", "Main Hall"); + const b = await add("places", "two", "Side hall"); + const first = await pickEntries(sql, SITE, { + q: "HALL", + collections: ["places", "halls"], + limit: 1, + }); + expect(first.entries.map((e) => e.id)).toEqual([a]); + expect(first.truncated).toBeUndefined(); + const second = await pickEntries(sql, SITE, { + q: "HALL", + collections: ["places", "halls"], + limit: 1, + cursor: first.cursor, + }); + expect(second.entries.map((e) => e.id)).toEqual([b]); + expect(second.entries[0]).toMatchObject({ collection: "places", locale: "tr", groupId: b }); + for (const q of ["#", "&", " ! "]) + expect( + (await pickEntries(sql, SITE, { q, collections: ["places"], limit: 5 })).entries, + ).toEqual([]); + expect( + (await pickEntries(sql, SITE, { q: "", collections: ["places"], limit: 5 })).entries, + ).toHaveLength(1); + }); + + it("visits ranges in key order: post-x before post, news2 before news, with and without a locale", async () => { + const { sql, add } = await world(open); + const ids = new Set(); + for (const c of ["post", "post-x", "news", "news2"]) { + ids.add(await add(c, `${c}-a`, `Hall in ${c}`)); + ids.add(await add(c, `${c}-b`, `Other hall in ${c}`)); + } + const collections = ["post", "post-x", "news", "news2"]; + for (const locale of [undefined, "tr"]) { + const one = await pickEntries(sql, SITE, { q: "hall", collections, locale, limit: 20 }); + expect(new Set(one.entries.map((e) => e.id))).toEqual(ids); + expect(one.cursor).toBeUndefined(); + // and across budget-truncated continuations, one row at a time + const found: string[] = []; + for (let cursor: string | undefined, guard = 0; guard < 50; guard++) { + const page = await pickEntries(sql, SITE, { + q: "hall", + collections, + locale, + limit: 20, + budget: 1, + cursor, + }); + found.push(...page.entries.map((e) => e.id)); + if (!page.cursor) break; + cursor = page.cursor; + } + expect(new Set(found)).toEqual(ids); + expect(found).toHaveLength(ids.size); + } + }); +}); + +describe("pickEntries' query plan (better-sqlite3)", () => { + it("reads each collection range through nodes_site_kind_key", async () => { + const sql = await BetterSqlite3Driver.open(":memory:"); + await migrate(sql, migrations); + const plan = await sql.all<{ detail: string }>(`EXPLAIN QUERY PLAN ${PICK_RANGE_SQL}`, [ + SITE, + "places:", + "", + "places;", + 10, + ]); + expect(plan.map((p) => p.detail).join("\n")).toMatch( + /USING INDEX nodes_site_kind_key \(site_id=\? AND kind=\? AND key>\? AND key<\?\)/, + ); + }); +}); diff --git a/packages/api/test/references-api.test.ts b/packages/api/test/references-api.test.ts new file mode 100644 index 00000000..b8180de1 --- /dev/null +++ b/packages/api/test/references-api.test.ts @@ -0,0 +1,32 @@ +import { SiteGraphStore } from "@cloudhal/core-ts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { D1Driver } from "@cloudhal/storage/d1"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { createMiniflareD1, createTableReset } from "@cloudhal/storage/testing"; +import { describeReferencesApi } from "../src/testing-references.ts"; + +describeReferencesApi("better-sqlite3", { + async create() { + const store = new SiteGraphStore({ + sql: await BetterSqlite3Driver.open(":memory:"), + blobs: new MemoryBlobStore(), + }); + await store.init(); + return store; + }, +}); + +let d1: Awaited> | undefined; +const reset = createTableReset(); +describeReferencesApi("Cloudflare D1 (Miniflare)", { + async create() { + d1 ??= await createMiniflareD1(); + const sql = new D1Driver(d1.db); + const store = new SiteGraphStore({ sql, blobs: new MemoryBlobStore() }); + await reset(sql, () => store.init()); + return store; + }, + async dispose() { + await d1?.dispose(); + }, +}); diff --git a/packages/cli/src/commands/import-describe.ts b/packages/cli/src/commands/import-describe.ts index b4d529fb..d2ca8321 100644 --- a/packages/cli/src/commands/import-describe.ts +++ b/packages/cli/src/commands/import-describe.ts @@ -297,6 +297,7 @@ export function describeImport(result: ImportArchiveResult): string[] { ...manifestLines(result), ...(plan ? planLines(result, plan) : []), ...replaceLines(result.replace, { dryRun: true, plan, from: "the archive" }), + ...(result.warnings ?? []), ...result.refusals.map((r) => `would refuse: ${r}`), "nothing written (dry run)", ]; diff --git a/packages/cli/src/commands/import-references.ts b/packages/cli/src/commands/import-references.ts new file mode 100644 index 00000000..030bc6ce --- /dev/null +++ b/packages/cli/src/commands/import-references.ts @@ -0,0 +1,55 @@ +/** + * RFC-0045 §2 "Archives" and OQ 14 for `cloudhal import --dry-run`: what an archive's references + * will show, reported, never refused. `referenceIssues` lists the references of entries shown now + * that will not resolve (a dangling or foreign group id, a draft or gated target, a missing + * language version); a relinked original (other nodes name it as their group while it belongs to + * another) stops lending its non-localized values to them. Pure over the bundle's rows. + */ +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import type { SiteBundle } from "@cloudhal/core-ts"; +import { referenceIssues } from "@cloudhal/core-ts/fields"; +import { displayText } from "./import-checks.ts"; + +const SHOWN = 20; + +/** Originals other entries use as their group whose own group is another node's. */ +function relinkedOriginals(entries: readonly ContentEntry[]): ContentEntry[] { + const byId = new Map(entries.map((e) => [e.id, e])); + const out = new Map(); + for (const e of entries) { + const group = e.translation?.group; + const original = group && group !== e.id ? byId.get(group) : undefined; + if (original && (original.translation?.group ?? original.id) !== original.id) + out.set(original.id, original); + } + return [...out.values()]; +} + +/** The warning lines of a dry run for the bundle's working copy (none when all is well). */ +export function referenceLines(bundle: SiteBundle, now: number): string[] { + const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + if (!siteRow) return []; + const site = JSON.parse(siteRow.body) as Site; + const entries = bundle.nodes + .filter((n) => n.kind === "contentEntry") + .map((n) => JSON.parse(n.body) as ContentEntry); + const lines: string[] = []; + const issues = referenceIssues({ site, entries }, now); + if (issues.length > 0) { + lines.push(`warning: ${issues.length} reference(s) of published entries will show nothing:`); + for (const i of issues.slice(0, SHOWN)) + lines.push( + ` ${i.entryId} (${i.locale}) ${i.path} → ${displayText(i.groupId, 48)}: ${i.reason}`, + ); + if (issues.length > SHOWN) lines.push(` … and ${issues.length - SHOWN} more`); + } + const relinked = relinkedOriginals(entries); + if (relinked.length > 0) + lines.push( + `warning: ${relinked.length} original(s) belong to another translation group, so their translations read none of their non-localized values: ${relinked + .slice(0, SHOWN) + .map((e) => e.id) + .join(", ")}`, + ); + return lines; +} diff --git a/packages/cli/src/commands/import.ts b/packages/cli/src/commands/import.ts index 4575ffb1..5cd7220c 100644 --- a/packages/cli/src/commands/import.ts +++ b/packages/cli/src/commands/import.ts @@ -77,6 +77,7 @@ import { } from "./import-checks.ts"; import { writeMedia } from "./import-media.ts"; import { archiveSize, readPass, siteSummary, statOrUndefined } from "./import-read.ts"; +import { referenceLines } from "./import-references.ts"; import { hostRefusal, importBracket, @@ -128,6 +129,8 @@ export interface ImportArchiveResult { readonly plan?: ImportPlan | undefined; /** Why the import is refused (a dry run reports them; an import throws the first). */ readonly refusals: readonly string[]; + /** RFC-0045: a dry run's reference warnings (`referenceLines`), never refusals. */ + readonly warnings?: readonly string[] | undefined; readonly imported?: | { readonly nodes: CopySiteResult["nodes"]; @@ -350,7 +353,10 @@ async function runImport(options: ImportArchiveOptions, state: State) { incoming: base.site.published, }) : undefined; - if (options.dryRun) return { ...base, plan, refusals, replace: outcome }; + if (options.dryRun) { + const warnings = referenceLines(first.bundle as SiteBundle, Date.now()); + return { ...base, plan, refusals, warnings, replace: outcome }; + } if (refusals[0] !== undefined) throw new ImportRefusal(refusals[0]); options.signal?.throwIfAborted(); const bundle = await copyMedia(run, first, plan); diff --git a/packages/cli/test/import-references.test.ts b/packages/cli/test/import-references.test.ts new file mode 100644 index 00000000..86799fe8 --- /dev/null +++ b/packages/cli/test/import-references.test.ts @@ -0,0 +1,62 @@ +/** RFC-0045 §2 "Archives", OQ 14: `cloudhal import --dry-run`'s reference warnings. */ +import type { SiteBundle } from "@cloudhal/core-ts"; +import { describe, expect, it } from "vitest"; +import { referenceLines } from "../src/commands/import-references.ts"; + +const SITE = "site_01HZX8K3Q9AB"; +const row = (kind: string, body: Record) => + ({ id: body.id, kind, site_id: SITE, body: JSON.stringify(body) }) as never; +const entry = (id: string, over: Record = {}) => + row("contentEntry", { + id, + collection: "events", + locale: "en", + status: "published", + fields: {}, + ...over, + }); + +function bundle(collections: unknown, entries: unknown[]): SiteBundle { + return { + siteId: SITE, + nodes: [row("site", { id: SITE, defaultLocale: "en", collections }), ...entries], + } as unknown as SiteBundle; +} + +describe("referenceLines", () => { + it("names the published references that will show nothing", () => { + const venue = { + key: "venue", + type: "reference", + labels: { en: { label: "V" } }, + localized: false, + collections: ["places"], + }; + const lines = referenceLines( + bundle({ events: { fields: [venue] }, places: {} }, [ + entry("entry_01HZX8K3QEVT1", { fields: { venue: "entry_01HZX8K3QGONE" } }), + ]), + 0, + ); + expect(lines[0]).toMatch(/1 reference\(s\)/); + expect(lines[1]).toContain("fields/venue → entry_01HZX8K3QGONE: missing"); + }); + + it("names relinked originals (OQ 14) and is quiet otherwise", () => { + const lines = referenceLines( + bundle({}, [ + entry("entry_01HZX8K3QORIG", { + translation: { group: "entry_01HZX8K3QOTHR", method: "human" }, + }), + entry("entry_01HZX8K3QTRAN", { + locale: "tr", + translation: { group: "entry_01HZX8K3QORIG", method: "human" }, + }), + ]), + 0, + ); + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("entry_01HZX8K3QORIG"); + expect(referenceLines(bundle({}, [entry("entry_01HZX8K3QALON")]), 0)).toEqual([]); + }); +}); diff --git a/packages/cloudflare/src/features.ts b/packages/cloudflare/src/features.ts index 30788f56..4f467427 100644 --- a/packages/cloudflare/src/features.ts +++ b/packages/cloudflare/src/features.ts @@ -6,6 +6,7 @@ * `expiry` until it answers `{ "gates": 2 }` (derived gates, RFC-0042 §3.6). */ import { AVAILABILITY_GATES_VERSION, type GateSupport } from "@cloudhal/core-ts/availability"; +import { FIELDS_FEATURE_VERSION, type FieldTypeSupport } from "@cloudhal/core-ts/fields"; import type { SiteFetch } from "./purge.ts"; const FEATURES_URL = "https://site.internal/__cloudhal/features"; @@ -14,7 +15,7 @@ const FEATURES_URL = "https://site.internal/__cloudhal/features"; export async function siteFeatures( fetch: SiteFetch, token: string, -): Promise<{ gates?: number } | undefined> { +): Promise<{ gates?: number; fields?: number } | undefined> { const res = await fetch(FEATURES_URL, { method: "GET", headers: { authorization: `Bearer ${token}` }, @@ -23,8 +24,13 @@ export async function siteFeatures( await res.body?.cancel(); return undefined; } - const body = (await res.json().catch(() => undefined)) as { gates?: unknown } | undefined; - return typeof body?.gates === "number" ? { gates: body.gates } : {}; + const body = (await res.json().catch(() => undefined)) as + | { gates?: unknown; fields?: unknown } + | undefined; + return { + ...(typeof body?.gates === "number" ? { gates: body.gates } : {}), + ...(typeof body?.fields === "number" ? { fields: body.fields } : {}), + }; } /** @@ -68,3 +74,36 @@ export function siteGateSupport(options: { }, }; } + +/** + * RFC-0045 §1: the store's reference and repeater capability for one admin Worker isolate, the + * twin of `siteGateSupport`: yes once the site Worker answers `{ "fields": 2 }` or more (kept as + * a plain value); a refusal or an error is asked again. Built in rollout step 2; nothing passes it + * to a store until step 7 enables the types. + */ +export function siteFieldTypeSupport(options: { + readonly fetch: SiteFetch | undefined; + readonly token: string | undefined; + readonly confirmed: () => boolean; + readonly remember: () => void; + readonly log?: (line: string) => void; +}): FieldTypeSupport { + return { + supported: async () => { + if (options.confirmed()) return true; + if (!options.fetch || !options.token) return false; + try { + const fields = (await siteFeatures(options.fetch, options.token))?.fields ?? 0; + if (fields < FIELDS_FEATURE_VERSION) { + options.log?.("site Worker does not read reference and repeater fields yet"); + return false; + } + options.remember(); + return true; + } catch (e) { + options.log?.(`features check failed: ${e instanceof Error ? e.message : String(e)}`); + return false; + } + }, + }; +} diff --git a/packages/cloudflare/test/features.test.ts b/packages/cloudflare/test/features.test.ts index 6a357998..feee1186 100644 --- a/packages/cloudflare/test/features.test.ts +++ b/packages/cloudflare/test/features.test.ts @@ -6,7 +6,7 @@ * packages/cli/test/cloudflare-gates.test.ts. */ import { describe, expect, it } from "vitest"; -import { siteFeatures, siteGateSupport } from "../src/features.ts"; +import { siteFeatures, siteFieldTypeSupport, siteGateSupport } from "../src/features.ts"; function site(answer: () => Response) { const calls: { url: string; auth: string | null; method: string | undefined }[] = []; @@ -102,3 +102,33 @@ describe("features handshake (admin side)", () => { expect(s.calls).toHaveLength(after); }); }); + +describe("fields: 2 (RFC-0045 §1, admin side)", () => { + it("says yes once the site Worker reports fields 2, remembers it, and asks again after a no", async () => { + let remembered = false; + let answer = { gates: 2, fields: 1 }; + const s = site(() => Response.json(answer)); + const support = siteFieldTypeSupport({ + fetch: s.fetch, + token: "tok", + confirmed: () => remembered, + remember: () => { + remembered = true; + }, + }); + expect(await support.supported("site_01HZX8K3Q9AB")).toBe(false); + answer = { gates: 2, fields: 2 }; + expect(await support.supported("site_01HZX8K3Q9AB")).toBe(true); + expect(remembered).toBe(true); + expect(await support.supported("site_01HZX8K3Q9AB")).toBe(true); + expect(s.calls).toHaveLength(2); + // no binding or token: refused, never asked + const none = siteFieldTypeSupport({ + fetch: undefined, + token: undefined, + confirmed: () => false, + remember: () => {}, + }); + expect(await none.supported("site_01HZX8K3Q9AB")).toBe(false); + }); +}); diff --git a/packages/core-ts/src/cross-site.ts b/packages/core-ts/src/cross-site.ts index 63276965..40aabc54 100644 --- a/packages/core-ts/src/cross-site.ts +++ b/packages/core-ts/src/cross-site.ts @@ -7,6 +7,7 @@ import type { ContentEntry, Page } from "@cloudhal/core-contracts"; import { mediaBlobKey } from "./assets.ts"; import { InvariantError } from "./errors.ts"; +import { type EditionDrop, editionReferences } from "./references/editions.ts"; import type { SiteGraphStore } from "./store.ts"; type Localized = Page | ContentEntry; @@ -111,7 +112,10 @@ export async function termsOnSite( return out as ContentEntry["terms"]; } -/** The node as written on the edition: its media copied, its terms the edition's. */ +/** + * The node as written on the edition: its media copied, its terms the edition's, and only the + * references the edition can resolve (RFC-0045 §3); `onDropped` hears what was not carried. + */ export async function forEdition( store: SiteGraphStore, node: T, @@ -119,10 +123,22 @@ export async function forEdition( fromSiteId: string; toSiteId: string; actor: Parameters[1]["actor"]; + onDropped?: ((dropped: EditionDrop[]) => void) | undefined; }, ): Promise { const ids = await copyAssetsToSite(store, { ...input, ids: assetIdsOf(node) }); const moved = withAssets({ ...node, siteId: input.toSiteId }, ids); if ("sections" in moved) return moved; - return { ...moved, terms: await termsOnSite(store, node as ContentEntry, input.toSiteId) } as T; + const entry = moved as ContentEntry; + const site = await store.getSite(input.toSiteId); + // A re-save of an existing edition translation: its stored values are not fresh. + const held = await store.getContentEntry(entry.id); + const stored = held?.siteId === input.toSiteId ? held : undefined; + const refs = site ? await editionReferences(store.sql, site, { entry, stored }) : undefined; + if (refs && refs.dropped.length > 0) input.onDropped?.(refs.dropped); + return { + ...entry, + ...(refs ? { fields: refs.fields ?? {} } : {}), + terms: await termsOnSite(store, node as ContentEntry, input.toSiteId), + } as T; } diff --git a/packages/core-ts/src/fields-operations/rewrite.ts b/packages/core-ts/src/fields-operations/rewrite.ts index a2c29278..e196eed8 100644 --- a/packages/core-ts/src/fields-operations/rewrite.ts +++ b/packages/core-ts/src/fields-operations/rewrite.ts @@ -6,6 +6,7 @@ * at the source's new hash in the same pass, guarded on that new hash (rule 4), so an operation * never makes translations look stale. `asset_refs` rows under `fields.*` follow each entry. */ + import { assertValid, type ContentEntry, @@ -13,12 +14,16 @@ import { type Site, } from "@cloudhal/core-contracts"; import type { SqlStatement } from "@cloudhal/storage"; +import { ConflictError } from "../errors.ts"; import type { ValueChange } from "../fields/migration.ts"; import { sameJson } from "../fields/types.ts"; import { nodeHash } from "../hash.ts"; import { passResetStatement } from "../list-values/state-sql.ts"; import { fieldAssetRefs } from "../media/field-refs.ts"; import { and, type Cond, casUpdateStatement, hashIs } from "../node-sql.ts"; +import { hasReferenceFields } from "../references/issues.ts"; +import { definitionsAnchor } from "../references/sql.ts"; +import { entryRefStatements, rewriteTargetGuard } from "../references/writer.ts"; import { fieldRefStatements } from "../store/entry-fields.ts"; import { forwardedSource } from "../store/source-forward.ts"; import { FIELD_OPERATION_BATCH, type OperationContext } from "./types.ts"; @@ -145,17 +150,29 @@ async function planPass(input: PassInput, ts: string) { return { planned: [...planned.values()], decisions }; } -/** The statements of one entry: its CAS, then its `fields.*` asset rows when values moved. */ -async function statementsOf(input: PassInput, p: Planned): Promise { +/** + * The statements of one entry: its CAS, then its `fields.*` asset rows and `entry_refs` rows + * when values moved. `anchor` is the pass's definitions anchor (sites with reference fields). + */ +async function statementsOf( + input: PassInput, + p: Planned, + anchor: Cond | undefined, +): Promise { + // RFC-0045 §2.6: a rewrite that changes reference values writes them only under the target guard. + const targets = p.fieldsChanged + ? rewriteTargetGuard(input.site, p.next, p.stored.entry) + : undefined; const cas = await casUpdateStatement( "contentEntry", p.next, p.stored.entry.rev, - and(input.guard, p.anchor), + and(input.guard, p.anchor, targets, anchor), ); if (!p.fieldsChanged) return [cas]; const refs = fieldAssetRefs(input.defs, p.next.fields); - return [cas, ...fieldRefStatements(input.site.id, p.next.id, refs, p.hash)]; + const rows = hasReferenceFields(input.site) ? entryRefStatements(input.site, p.next, p.hash) : []; + return [cas, ...fieldRefStatements(input.site.id, p.next.id, refs, p.hash), ...rows]; } const outOfBudget = (ctx: OperationContext, done: number) => @@ -177,6 +194,20 @@ export async function rewritePass(input: PassInput): Promise { const { ctx } = input; const { planned, decisions } = await planPass(input, ctx.stamp()); const out: PassResult = { rewritten: 0, skipped: [], written: [], exhausted: false }; + // RFC-0045 §2.1: rows and target pairs come from the pass's definitions, so anchor them. + let anchor: Cond | undefined; + if (hasReferenceFields(input.site)) + try { + anchor = await definitionsAnchor(ctx.store.sql, input.site, input.lists.collection); + } catch (e) { + if (!(e instanceof ConflictError)) throw e; + for (const p of planned) + out.skipped.push({ + entryId: p.next.id, + reason: "the site changed during the operation; retried", + }); + return out; + } let i = 0; while (i < planned.length) { if (outOfBudget(ctx, input.done + out.rewritten)) { @@ -189,7 +220,7 @@ export async function rewritePass(input: PassInput): Promise { while (i < planned.length && statements.length < FIELD_OPERATION_BATCH && heads.length < room) { const p = planned[i++] as Planned; heads.push({ index: statements.length, p }); - statements.push(...(await statementsOf(input, p))); + statements.push(...(await statementsOf(input, p, anchor))); } const batch = heads.map((h) => h.p); const ids = batch.map((p) => p.next.id); diff --git a/packages/core-ts/src/fields/index.ts b/packages/core-ts/src/fields/index.ts index a280d869..534ef014 100644 --- a/packages/core-ts/src/fields/index.ts +++ b/packages/core-ts/src/fields/index.ts @@ -11,6 +11,8 @@ */ export { + FIELDS_FEATURE_VERSION, + FIELDS_SUPPORTED, type FieldTypeSupport, type ReferenceDefinitionOptions, referenceDefinitionIssues, diff --git a/packages/core-ts/src/media/field-refs.ts b/packages/core-ts/src/media/field-refs.ts index e47a3a8c..fed713cc 100644 --- a/packages/core-ts/src/media/field-refs.ts +++ b/packages/core-ts/src/media/field-refs.ts @@ -8,27 +8,58 @@ */ import type { Asset, FieldDefinition } from "@cloudhal/core-contracts"; import { ASSET_ID_PATTERN } from "../fields/props.ts"; -import type { FieldIssue } from "../fields/types.ts"; +import { type FieldIssue, isRecord } from "../fields/types.ts"; import { type AssetRefUse, markdownAssetIds } from "./refs.ts"; const ASSET_ID = new RegExp(ASSET_ID_PATTERN); -/** Every asset reference of an entry's field values under `defs`, sorted like `assetRefsOf`. */ +/** + * A field's asset reference with where it sits: `fields/cover`, or inside a repeater item + * `fields/speakers/2/photo` (RFC-0045 §2.5), so issues map to their control. + */ +export interface FieldAssetRef extends AssetRefUse { + /** The first use's path. */ + readonly path: string; + /** Every use's path, in order (one asset in several items is one row, but several uses). */ + readonly paths: readonly string[]; +} + +type Slot = { type: string }; + +/** One value's assets: an image's id, a rich text's `asset:` links. */ +function valueAssets(slot: Slot, value: unknown): string[] { + if (typeof value !== "string") return []; + if (slot.type === "image") return ASSET_ID.test(value) ? [value] : []; + return slot.type === "richText" && value.includes("asset:") ? markdownAssetIds(value) : []; +} + +/** + * Every asset reference of an entry's field values under `defs`, sorted like `assetRefsOf`: top + * level under `fields.`, and in repeater items under `fields..` (one row per asset + * and field; the first use's item path is kept). + */ export function fieldAssetRefs( defs: readonly FieldDefinition[], values: Readonly> | undefined, -): AssetRefUse[] { +): FieldAssetRef[] { if (!values || defs.length === 0) return []; - const out = new Map(); - const add = (assetId: string, field: string) => - out.set(`${assetId}\n${field}`, { assetId, field }); + const out = new Map(); + const add = (assetId: string, field: string, path: string) => { + const id = `${assetId}\n${field}`; + const held = out.get(id); + if (held) held.paths.push(path); + else out.set(id, { assetId, field, path, paths: [path] }); + }; for (const def of defs) { - const value = values[def.key]; - if (typeof value !== "string") continue; - const field = `fields.${def.key}`; - if (def.type === "image" && ASSET_ID.test(value)) add(value, field); - else if (def.type === "richText" && value.includes("asset:")) - for (const id of markdownAssetIds(value)) add(id, field); + const value = Object.hasOwn(values, def.key) ? values[def.key] : undefined; + for (const id of valueAssets(def, value)) add(id, `fields.${def.key}`, `fields/${def.key}`); + if (def.type !== "repeater" || !Array.isArray(value)) continue; + value.forEach((item, i) => { + if (!isRecord(item)) return; + for (const sub of def.fields) + for (const id of valueAssets(sub, Object.hasOwn(item, sub.key) ? item[sub.key] : undefined)) + add(id, `fields.${def.key}.${sub.key}`, `fields/${def.key}/${i}/${sub.key}`); + }); } return [...out.values()].sort( (a, b) => a.assetId.localeCompare(b.assetId) || a.field.localeCompare(b.field), @@ -73,55 +104,82 @@ export interface FieldAssetCheck { /** The entry's field definitions (its new collection's). */ readonly defs: readonly FieldDefinition[]; /** `fieldAssetRefs` of the values being written. */ - readonly refs: readonly AssetRefUse[]; + readonly refs: readonly UseRef[]; /** `fieldAssetRefs` of the stored entry (empty for a new one). */ readonly kept: readonly AssetRefUse[]; /** Assets of the entry's site, by id. */ readonly lookup: SiteAssetLookup; - /** Paths (`fields/`) whose value already has an issue: their references are not checked. */ + /** + * Issue paths (`fields/`, `fields/speakers/2/person`) whose value already has an issue: + * a reference is not checked when one of these is a prefix of its path, or it of one. + */ readonly skip?: ReadonlySet; } /** References a write adds: not held by the stored entry under the same field. */ -export function newFieldAssetRefs( - refs: readonly AssetRefUse[], +export function newFieldAssetRefs( + refs: readonly R[], kept: readonly AssetRefUse[], -): AssetRefUse[] { +): R[] { const held = new Set(kept.map((r) => `${r.assetId}\n${r.field}`)); return refs.filter((r) => !held.has(`${r.assetId}\n${r.field}`)); } +/** A reference with where it is used: `fieldAssetRefs`' item paths, or none (a top-level ref). */ +type UseRef = AssetRefUse & { readonly path?: string; readonly paths?: readonly string[] }; + +/** A ref's use paths, else its field's (`fields.` → `fields/`). */ +const pathsOf = (r: UseRef): readonly string[] => + r.paths ?? [r.path ?? `fields/${r.field.slice("fields.".length)}`]; + +/** One path lies at or under the other (`fields/speakers` and `fields/speakers/2/photo`). */ +const nested = (a: string, b: string) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`); + +/** Field and sub-field types by `` and `.`. */ +function slotTypes(defs: readonly FieldDefinition[]): Map { + const out = new Map(); + for (const d of defs) { + out.set(d.key, d.type); + if (d.type === "repeater") for (const s of d.fields) out.set(`${d.key}.${s.key}`, s.type); + } + return out; +} + /** - * RFC-0041 §2: every reference a write adds names a file of the site; an image field's is an - * image; a retired file is not newly used unless the entry already used it. Issues are - * `invalid_value` at `fields/`. + * RFC-0041 §2: every reference a write adds names a file of the site; an image field's (or image + * sub-field's) is an image; a retired file is not newly used unless the entry already used it. + * Issues are `invalid_value` at the reference's path (`fields/`, or an item path). */ export async function fieldAssetIssues(check: FieldAssetCheck): Promise { const { defs, kept, skip } = check; - const refs = newFieldAssetRefs(check.refs, kept).filter( - (r) => !skip?.has(r.field.replace(".", "/")), + const skipped = [...(skip ?? [])]; + // Every use no skip path covers: one asset in several items is reported at each item. + const uses = newFieldAssetRefs(check.refs, kept).flatMap((ref) => + pathsOf(ref) + .filter((path) => !skipped.some((s) => nested(path, s))) + .map((path) => ({ ref, path })), ); - if (refs.length === 0) return []; + if (uses.length === 0) return []; + const refs = [...new Set(uses.map((u) => u.ref))]; const assets = await check.lookup(refs.map((r) => r.assetId)); const keptIds = new Set(kept.map((r) => r.assetId)); - const types = new Map(defs.map((d) => [d.key, d.type])); + const types = slotTypes(defs); const out: FieldIssue[] = []; - for (const ref of refs) { - const key = ref.field.slice("fields.".length); - const problem = assetProblem(assets.get(ref.assetId), types.get(key), keptIds.has(ref.assetId)); - if (problem) - out.push({ - path: `fields/${key}`, - code: "invalid_value", - message: `${ref.assetId} ${problem}`, - }); + for (const { ref, path } of uses) { + const slot = ref.field.slice("fields.".length); + const problem = assetProblem( + assets.get(ref.assetId), + types.get(slot), + keptIds.has(ref.assetId), + ); + if (problem) out.push({ path, code: "invalid_value", message: `${ref.assetId} ${problem}` }); } return out; } function assetProblem( asset: Asset | undefined, - type: FieldDefinition["type"] | undefined, + type: string | undefined, kept: boolean, ): string | undefined { if (!asset) return "is not a file of this site"; diff --git a/packages/core-ts/src/references/definitions.ts b/packages/core-ts/src/references/definitions.ts index aa7fed94..d53bbb8a 100644 --- a/packages/core-ts/src/references/definitions.ts +++ b/packages/core-ts/src/references/definitions.ts @@ -26,6 +26,18 @@ export interface FieldTypeSupport { supported(siteId: string): boolean | Promise; } +/** + * The `fields` value of `GET /__cloudhal/features` from which a site Worker reads reference and + * repeater fields (rollout step 3 reports it; step 7 starts asking it). + */ +export const FIELDS_FEATURE_VERSION = 2; + +/** + * The renderer in this process reads both types (Node: `cloudhal dev`, `studio`, self-host), the + * twin of `GATES_ENFORCED`. Not passed by any production caller before rollout step 7. + */ +export const FIELDS_SUPPORTED: FieldTypeSupport = { supported: () => true }; + /** True only when a capability is given and says yes for the site. */ export async function referenceTypesSupported( siteId: string, diff --git a/packages/core-ts/src/references/editions.ts b/packages/core-ts/src/references/editions.ts new file mode 100644 index 00000000..5df9027a --- /dev/null +++ b/packages/core-ts/src/references/editions.ts @@ -0,0 +1,199 @@ +/** + * RFC-0045 §3 "Language editions": the references a node keeps when it is written on another site + * of the project. Group ids are shared across sites, so a copied id stays the same id. What is + * kept is exactly what the edition's own write accepts (§2.1), so the edition write never fails + * on what the edition cannot carry: + * + * - a group with a member of the edition site (membership rule, not archived) in a collection the + * edition's field allows, meeting the field's `where` (every value of a copy is newly chosen); + * - for a single-target field, the first such id; for a multiple one, at most `maxItems`; + * - at most 100 references per entry across fields and items; + * - a repeater item left with no value is dropped, and at most `maxItems` items are kept. + * + * Values are read through a running rename's `migration.fromKey` and written back under the key + * they were read from (`storedKey`, as `entryRefRows` reads them). Every value not carried is + * reported with its path and why. A `requiredToPublish` field emptied this way is fine: the copy + * is a draft. + */ +import type { ContentEntry, FieldCondition, FieldDefinition } from "@cloudhal/core-contracts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { + FIELD_LIMITS, + fieldDefinitions, + isRecord, + type SiteFields, + sameJson, +} from "../fields/types.ts"; +import { entryRefRows, holdsReferences, storedKey } from "./rows.ts"; +import { readMembers, targetProblem } from "./targets.ts"; +import { referenceUses } from "./values.ts"; + +/** Why a value was not carried. */ +export type EditionDropReason = "missing" | "where" | "single" | "maxItems" | "cap" | "empty"; + +/** A value an edition copy did not carry: a reference (`groupId`) or a whole item. */ +export interface EditionDrop { + /** `fields/venue`, `fields/speakers/1/person`, or an item `fields/speakers/1`. */ + path: string; + groupId?: string; + reason: EditionDropReason; +} + +interface Target { + collections: readonly string[]; + where?: readonly FieldCondition[] | undefined; +} + +/** Whether the edition takes `groupId` for a target, and why not. */ +type Usable = (groupId: string, target: Target) => "missing" | "where" | undefined; + +interface Carry { + usable: Usable; + dropped: EditionDrop[]; + /** References still allowed in this entry (the per-entry cap). */ + budget: { left: number }; +} + +/** One reference id: kept, or dropped with its reason. */ +function keep(id: unknown, at: { path: string; target: Target; carry: Carry }): id is string { + const { path, target, carry } = at; + const why = + typeof id !== "string" + ? "missing" + : (carry.usable(id, target) ?? (carry.budget.left > 0 ? undefined : "cap")); + if (why === undefined) { + carry.budget.left--; + return true; + } + carry.dropped.push({ path, ...(typeof id === "string" ? { groupId: id } : {}), reason: why }); + return false; +} + +function carryReference( + def: Extract, + value: unknown, + at: { path: string; carry: Carry }, +): unknown { + const { path, carry } = at; + const ids = Array.isArray(value) ? value : [value]; + const pathOf = (i: number) => (Array.isArray(value) ? `${path}/${i}` : path); + const target = { collections: def.collections, where: def.where }; + const kept: string[] = []; + const max = def.multiple ? (def.maxItems ?? FIELD_LIMITS.listItems) : 1; + ids.forEach((id, i) => { + if (kept.length >= max) { + const reason = def.multiple ? "maxItems" : "single"; + carry.dropped.push({ + path: pathOf(i), + ...(typeof id === "string" ? { groupId: id } : {}), + reason, + }); + return; + } + if (keep(id, { path: pathOf(i), target, carry })) kept.push(id); + }); + if (kept.length === 0) return undefined; + return def.multiple ? kept : kept[0]; +} + +function carryRepeater( + def: Extract, + value: unknown, + at: { path: string; carry: Carry }, +): unknown { + const { path, carry } = at; + if (!Array.isArray(value)) return value; + const max = def.maxItems ?? FIELD_LIMITS.listItems; + const items: unknown[] = []; + value.forEach((item, i) => { + if (items.length >= max) { + carry.dropped.push({ path: `${path}/${i}`, reason: "maxItems" }); + return; + } + if (!isRecord(item)) { + items.push(item); + return; + } + const next: Record = { ...item }; + for (const sub of def.fields) { + if (sub.type !== "reference" || next[sub.key] === undefined) continue; + const target = { collections: sub.collections }; + if (!keep(next[sub.key], { path: `${path}/${i}/${sub.key}`, target, carry })) + delete next[sub.key]; + } + if (Object.keys(next).length > 0) items.push(next); + else carry.dropped.push({ path: `${path}/${i}`, reason: "empty" }); + }); + return items.length > 0 ? items : undefined; +} + +/** The edition's members of `ids` (any status), and the write's own target rule over them. */ +async function usableOn( + sql: SqlDriver, + site: SiteFields & { id: string }, + ids: readonly string[], +): Promise<(groupId: string, target: Target, fresh: boolean) => "missing" | "where" | undefined> { + const members = await readMembers(sql, site.id, ids); + return (groupId, target, fresh) => { + const problem = targetProblem(site, { groupId, ...target, fresh }, members); + return problem === undefined ? undefined : problem === "where" ? "where" : "missing"; + }; +} + +/** The group ids a stored value holds for `def`: the freshness test only (not a count). */ +const heldBy = (def: FieldDefinition, value: unknown): Set => + new Set(referenceUses(def, value, "").map((u) => u.groupId)); + +/** + * `entry`'s fields as written on site `toSite` (its own definitions), with what was dropped. + * One read of the edition's members of the named groups. + */ +export async function editionReferences( + sql: SqlDriver, + toSite: SiteFields & { id: string }, + input: { + entry: Pick; + /** The edition's stored version of the entry (a re-save): its values are not fresh. */ + stored?: Pick | undefined; + }, +): Promise<{ fields: ContentEntry["fields"]; dropped: EditionDrop[] }> { + const { entry } = input; + const defs = fieldDefinitions(toSite, entry.collection).filter(holdsReferences); + const ids = [...new Set(entryRefRows(defs, entry.fields).map((r) => r.t))]; + if (ids.length === 0) return { fields: entry.fields, dropped: [] }; + const usable = await usableOn(sql, toSite, ids); + const before: Record = + input.stored?.collection === entry.collection ? { ...(input.stored.fields ?? {}) } : {}; + const budget = { left: FIELD_LIMITS.referencesPerEntry }; + const dropped: EditionDrop[] = []; + const fields: Record = { ...(entry.fields ?? {}) }; + // Pass 1: unchanged values are the write's own business (it re-checks nothing unchanged), but + // every one of their references counts toward the per-entry cap, before any changed key. + const changed: { def: FieldDefinition; key: string }[] = []; + for (const def of defs) { + const key = storedKey(def, fields); + if (key === undefined) continue; + if (Object.hasOwn(before, key) && sameJson(before[key], fields[key])) + budget.left -= referenceUses(def, fields[key], "").length; + else changed.push({ def, key }); + } + // Pass 2: the changed keys, carried under what the cap leaves. + for (const { def, key } of changed) { + const held = Object.hasOwn(before, key) ? heldBy(def, before[key]) : new Set(); + const carry: Carry = { + usable: (g, target) => usable(g, target, !held.has(g)), + dropped, + budget, + }; + const at = { path: `fields/${key}`, carry }; + const value = + def.type === "reference" + ? carryReference(def, fields[key], at) + : def.type === "repeater" + ? carryRepeater(def, fields[key], at) + : fields[key]; + if (value === undefined) delete fields[key]; + else fields[key] = value; + } + return { fields, dropped }; +} diff --git a/packages/core-ts/src/references/rows.ts b/packages/core-ts/src/references/rows.ts index 47580554..11a1f47a 100644 --- a/packages/core-ts/src/references/rows.ts +++ b/packages/core-ts/src/references/rows.ts @@ -32,7 +32,10 @@ export const definitionsPath = (collection: string): string => `$.collections."${collection}".fields`; /** The key `def`'s value is stored under in `values`: its own, else a running rename's old one. */ -function storedKey(def: FieldDefinition, values: Record): string | undefined { +export function storedKey( + def: FieldDefinition, + values: Record, +): string | undefined { if (Object.hasOwn(values, def.key)) return def.key; const from = def.migration?.fromKey; return from !== undefined && Object.hasOwn(values, from) ? from : undefined; diff --git a/packages/core-ts/src/references/targets.ts b/packages/core-ts/src/references/targets.ts new file mode 100644 index 00000000..a992d69d --- /dev/null +++ b/packages/core-ts/src/references/targets.ts @@ -0,0 +1,86 @@ +/** + * RFC-0045 §2.1: whether a group id is a usable target for a field, the one rule the entry + * writer's plan-time check and the language-edition mapping share, so they cannot drift. + * + * Members are read without a status filter (an archived same-site original still lends its + * values to the translations that resolve through it). A target needs a member in one of the + * field's collections; a *fresh* value (newly chosen) also needs one that is not archived and, + * for a field with `where`, one that meets it. Pure but for `readMembers`. + */ +import type { ContentEntry, FieldCondition } from "@cloudhal/core-contracts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { matchesFieldQuery, parseFieldQuery } from "../fields/query.ts"; +import { resolveFields } from "../fields/resolve.ts"; +import type { SiteFields } from "../fields/types.ts"; +import { memberGroup } from "./members.ts"; + +export interface MemberRow { + id: string; + group: string; + collection: string; + status: string; + body: ContentEntry; +} + +/** The members of `groups` (and the nodes named by id) in this site, one read, any status. */ +export async function readMembers( + sql: SqlDriver, + siteId: string, + ids: readonly string[], +): Promise { + if (ids.length === 0) return []; + const rows = await sql.all<{ id: string; body: string }>( + `SELECT n.id AS id, n.body AS body FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' + AND (n.id IN (SELECT value FROM json_each(?)) + OR json_extract(n.body, '$.translation.group') IN (SELECT value FROM json_each(?)))`, + [siteId, JSON.stringify(ids), JSON.stringify(ids)], + ); + return rows.map((r) => { + const body = JSON.parse(r.body) as ContentEntry; + return { + id: r.id, + group: memberGroup(body), + collection: body.collection, + status: body.status, + body, + }; + }); +} + +/** One value to check: its group, the field's collections and `where`, and whether it is new. */ +export interface TargetCheck { + groupId: string; + collections: readonly string[]; + where?: readonly FieldCondition[] | undefined; + /** Newly chosen: its member must not be archived, and must meet `where`. */ + fresh: boolean; +} + +/** + * Why `check` fails against `members`: `absent` (no member in this site), `collection` (members + * only in collections the field does not allow), `archived` (a fresh value whose allowed members + * are all archived), `where` (none meets the field's conditions); undefined when one satisfies it. + */ +export function targetProblem( + site: SiteFields, + check: TargetCheck, + members: readonly MemberRow[], +): "absent" | "collection" | "archived" | "where" | undefined { + const of = members.filter((m) => m.group === check.groupId); + const allowed = of.filter((m) => check.collections.includes(m.collection)); + const usable = allowed.filter((m) => !check.fresh || m.status !== "archived"); + if (usable.length === 0) { + if (of.length === 0) return "absent"; + return allowed.length === 0 ? "collection" : "archived"; + } + const [target] = check.collections; + if (!check.fresh || !check.where || target === undefined) return undefined; + const parsed = parseFieldQuery(site, target, { where: [...check.where] }); + if (!parsed.ok) return undefined; + const byId = new Map(members.map((m) => [m.id, m.body])); + // `where` cannot use `now` (§1), so the instant is irrelevant. + const meets = usable.some((m) => + matchesFieldQuery(resolveFields(site, m.body, byId.get(m.group)), parsed.query, 0), + ); + return meets ? undefined : "where"; +} diff --git a/packages/core-ts/src/references/writer.ts b/packages/core-ts/src/references/writer.ts index c6e6f55a..b613f850 100644 --- a/packages/core-ts/src/references/writer.ts +++ b/packages/core-ts/src/references/writer.ts @@ -18,52 +18,18 @@ */ import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; import type { SqlDriver, SqlStatement } from "@cloudhal/storage"; -import { matchesFieldQuery, parseFieldQuery } from "../fields/query.ts"; import { refuseFieldIssues } from "../fields/refusal.ts"; -import { resolveFields } from "../fields/resolve.ts"; import { type FieldIssue, fieldDefinitions, isRecord, sameJson } from "../fields/types.ts"; import { and, type Cond } from "../node-sql.ts"; import { hasReferenceFields } from "./issues.ts"; import { memberGroup } from "./members.ts"; import { definitionsPath, entryRefRows, holdsReferences } from "./rows.ts"; import { definitionsAnchor, type TargetPair, targetGuard } from "./sql.ts"; +import { type MemberRow, readMembers, targetProblem } from "./targets.ts"; import { type ReferenceUse, referenceUses } from "./values.ts"; const ANCHOR = "EXISTS (SELECT 1 FROM nodes WHERE id = ? AND hash = ?)"; -interface MemberRow { - id: string; - group: string; - collection: string; - status: string; - body: ContentEntry; -} - -/** The members of `groups` (and the nodes named by id) in this site, one read. */ -async function readMembers( - sql: SqlDriver, - siteId: string, - ids: readonly string[], -): Promise { - if (ids.length === 0) return []; - const rows = await sql.all<{ id: string; body: string }>( - `SELECT n.id AS id, n.body AS body FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' - AND (n.id IN (SELECT value FROM json_each(?)) - OR json_extract(n.body, '$.translation.group') IN (SELECT value FROM json_each(?)))`, - [siteId, JSON.stringify(ids), JSON.stringify(ids)], - ); - return rows.map((r) => { - const body = JSON.parse(r.body) as ContentEntry; - return { - id: r.id, - group: memberGroup(body), - collection: body.collection, - status: body.status, - body, - }; - }); -} - /** Every group id a body's values name under `defs`. */ const namedIds = (defs: readonly FieldDefinition[], values: unknown): string[] => [ ...new Set(entryRefRows(defs, values).map((r) => r.t)), @@ -156,48 +122,35 @@ function guardedUses( return out; } -/** Why `use` fails against the members read; undefined when a member satisfies it. */ +/** Why `use` fails against the members read (`targetProblem`); undefined when it does not. */ function useIssue(site: Site, use: Use, members: readonly MemberRow[]): FieldIssue | undefined { - const of = members.filter((m) => m.group === use.groupId); - const allowed = of.filter((m) => use.collections.includes(m.collection)); - const usable = allowed.filter((m) => !use.fresh || m.status !== "archived"); - if (usable.length === 0) { + const where = use.def.type === "reference" ? use.def.where : undefined; + const problem = targetProblem(site, { ...use, where }, members); + const issue = (code: FieldIssue["code"], message: string): FieldIssue => ({ + path: use.path, + code, + message, + }); + switch (problem) { + case undefined: + return undefined; // A write naming an entry outside the field's collections: a value to fix (422), not the // move race's 409 `reference_target_moved`. - if (of.length > 0 && allowed.length === 0) - return { - path: use.path, - code: "unknown_entry", - message: `${use.groupId} has no entry in ${use.collections.join(", ")} of this site; pick one from those collections`, - }; - return { - path: use.path, - code: "unknown_entry", - message: - allowed.length > 0 - ? `${use.groupId} is archived; choose an entry that is not` - : `${use.groupId} is not an entry of this site`, - }; + case "collection": + return issue( + "unknown_entry", + `${use.groupId} has no entry in ${use.collections.join(", ")} of this site; pick one from those collections`, + ); + case "archived": + return issue("unknown_entry", `${use.groupId} is archived; choose an entry that is not`); + case "where": + return issue( + "invalid_value", + `${use.groupId} does not meet the conditions of ${use.def.key}`, + ); + default: + return issue("unknown_entry", `${use.groupId} is not an entry of this site`); } - const where = use.def.type === "reference" && use.fresh ? use.def.where : undefined; - if (!where) return undefined; - const target = use.collections[0] as string; - const parsed = parseFieldQuery(site, target, { where }); - if (!parsed.ok) return undefined; - const byId = new Map(members.map((m) => [m.id, m.body])); - const meets = usable.some((m) => { - const original = byId.get(m.group); - const resolved = resolveFields(site, m.body, original); - // `where` cannot use `now` (§1), so the instant is irrelevant. - return matchesFieldQuery(resolved, parsed.query, 0); - }); - return meets - ? undefined - : { - path: use.path, - code: "invalid_value", - message: `${use.groupId} does not meet the conditions of ${use.def.key}`, - }; } /** The plan-time checks of the guarded uses: own group, targets, `where`. */ @@ -259,6 +212,31 @@ export function entryRefStatements(site: Site, draft: ContentEntry, hash: string return out; } +/** The guard pairs of the uses a write checks (changed keys; all own values going live). */ +function pairsOf(uses: readonly Use[]): TargetPair[] { + const pairs = new Map(); + for (const u of uses) { + const pair: TargetPair = { t: u.groupId, c: u.collections, f: u.fresh ? 1 : 0 }; + pairs.set(JSON.stringify(pair), pair); + } + return [...pairs.values()]; +} + +/** + * §2.6: the target guard of a field-operation rewrite, over the reference values it changes + * (undefined when none change). A miss skips the entry for the next pass, like a concurrent edit. + * The rewrite's own statements are guarded on the site still carrying the operation's marker, so + * the definitions the pairs come from cannot change under it. + */ +export function rewriteTargetGuard( + site: Site, + next: ContentEntry, + stored: ContentEntry, +): Cond | undefined { + const defs = fieldDefinitions(site, next.collection); + return targetGuard(site.id, pairsOf(guardedUses(defs, next, stored))); +} + /** True when the write changes the entry's values or collection (the definitions anchor applies). */ const changesFields = (draft: ContentEntry, stored: ContentEntry | undefined) => !stored || stored.collection !== draft.collection || !sameJson(stored.fields, draft.fields); @@ -289,13 +267,8 @@ export async function planEntryRefs( const check = async () => refuseFieldIssues(useIssues(site, draft, uses, await readMembers(sql, site.id, groups))); await check(); - const pairs = new Map(); - for (const u of uses) { - const pair: TargetPair = { t: u.groupId, c: u.collections, f: u.fresh ? 1 : 0 }; - pairs.set(JSON.stringify(pair), pair); - } return { - guard: and(anchor, targetGuard(site.id, [...pairs.values()])), + guard: and(anchor, targetGuard(site.id, pairsOf(uses))), statements: (hash) => entryRefStatements(site, draft, hash), diagnose: check, }; diff --git a/packages/core-ts/src/release-deps.ts b/packages/core-ts/src/release-deps.ts new file mode 100644 index 00000000..e4170d7e --- /dev/null +++ b/packages/core-ts/src/release-deps.ts @@ -0,0 +1,49 @@ +/** + * RFC-0045 §2.7: the kind-aware dependency extractor of releases. References are not release + * dependencies: closure dependencies are hard (they lock and fail), references are soft at + * render. So reference values, top level and inside repeater items, are dropped from an entry's + * body by the site's definitions as pinned (read through a running rename's `fromKey`) before + * RFC-0008's generic `referencedNodeIds` walks it. Without the change a group id would be pulled + * into the closure and locked, and an edition's foreign group id would fail the run as + * `dependency_missing`. Other kinds and other values are walked as before. Worker-safe. + */ +import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { type NodeVersion, referencedNodeIds } from "@cloudhal/schedules/engine"; +import { fieldDefinitions, isRecord } from "./fields/types.ts"; + +/** One value with its reference ids removed (a repeater keeps its other sub-values). */ +function withoutReferences(def: FieldDefinition, value: unknown): unknown { + if (def.type === "reference") return undefined; + if (def.type !== "repeater" || !Array.isArray(value)) return value; + const refs = def.fields.filter((s) => s.type === "reference").map((s) => s.key); + if (refs.length === 0) return value; + return value.map((item) => + isRecord(item) + ? Object.fromEntries(Object.entries(item).filter(([k]) => !refs.includes(k))) + : item, + ); +} + +/** An entry body without its reference values, under the site's definitions. */ +export function stripReferences(site: Pick, body: unknown): unknown { + if (!isRecord(body) || typeof body.collection !== "string" || !isRecord(body.fields)) return body; + const defs = fieldDefinitions(site, body.collection); + if (!defs.some((d) => d.type === "reference" || d.type === "repeater")) return body; + const fields: Record = { ...body.fields }; + for (const def of defs) + for (const key of [def.key, def.migration?.fromKey]) + if (key !== undefined && Object.hasOwn(fields, key)) + fields[key] = withoutReferences(def, fields[key]); + return { ...body, fields }; +} + +/** `depsOf` for `dependencyClosure`: the ids a node version needs live, references excepted. */ +export function releaseDepsOf( + site: Pick, + bodyOf: (ref: NodeVersion) => Promise, +): (ref: NodeVersion) => Promise { + return async (ref) => { + const body = JSON.parse(await bodyOf(ref)) as unknown; + return referencedNodeIds(ref.kind === "contentEntry" ? stripReferences(site, body) : body); + }; +} diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index ea388661..731be999 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -48,7 +48,6 @@ import { type PinnedItem, pinItems, RELEASE_LEASE_MS, - referencedNodeIds, releaseWhen, runClaimedRelease, runDue, @@ -74,6 +73,7 @@ import { NotFoundError } from "./errors.ts"; import type { NodeHold } from "./holds.ts"; import * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; +import { releaseDepsOf } from "./release-deps.ts"; import { releaseExecutor } from "./release-executor.ts"; import { SiteGraphStore, type SiteGraphStoreOptions } from "./store.ts"; @@ -468,7 +468,7 @@ export class ReleaseService { items, working, live: liveRefs, - depsOf: async (ref) => referencedNodeIds(JSON.parse(await bodyOf(ref))), + depsOf: releaseDepsOf(site, bodyOf), // RFC-0045 §2.7: references are not dependencies blobOf: async (ref) => { const a = JSON.parse(await bodyOf(ref)) as { storage?: { kind?: string }; sha256?: string }; return a.storage?.kind === "blob" ? a.sha256 : undefined; diff --git a/packages/core-ts/src/store/entry-fields.ts b/packages/core-ts/src/store/entry-fields.ts index 02763deb..0b7e6ef0 100644 --- a/packages/core-ts/src/store/entry-fields.ts +++ b/packages/core-ts/src/store/entry-fields.ts @@ -106,7 +106,8 @@ export async function planEntryFields( const kept = stored ? fieldAssetRefs(fieldDefinitions(site, stored.collection), stored.fields) : []; - const skip = new Set(issues.map((i) => i.path.split("/").slice(0, 2).join("/"))); + // RFC-0045 §2.5: issue paths as they are; `fieldAssetIssues` matches them by prefix. + const skip = new Set(issues.map((i) => i.path)); const lookup = (ids: readonly string[]) => ctx.assets.getAssets(site.id, ids); issues.push(...(await fieldAssetIssues({ defs, refs, kept, lookup, skip }))); issues.push(...checkRequiredToPublish(site, draft, original)); diff --git a/packages/core-ts/src/testing-references.ts b/packages/core-ts/src/testing-references.ts index d5792adf..ede7853e 100644 --- a/packages/core-ts/src/testing-references.ts +++ b/packages/core-ts/src/testing-references.ts @@ -13,9 +13,14 @@ import type { Actor, ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; import { describe, expect, it } from "vitest"; import { GATES_ENFORCED } from "./availability.ts"; +import { forEdition } from "./cross-site.ts"; import { ConflictError, InvariantError } from "./errors.ts"; import { runFieldOperation } from "./fields-operations/index.ts"; +import { loadEntries, rewritePass } from "./fields-operations/rewrite.ts"; +import { operationContext } from "./fields-operations/types.ts"; import { canonicalJson, nodeHash } from "./hash.ts"; +import { PNG_1X1 } from "./media/fixtures.ts"; +import { editionReferences } from "./references/editions.ts"; import { SiteGraphStore } from "./store.ts"; import { exportSite, importSite } from "./sync.ts"; import { racingDriver, type SuiteContext } from "./testing-taxonomy.ts"; @@ -46,6 +51,7 @@ const SPEAKERS: FieldDefinition = { fields: [ { key: "person", type: "reference", labels: L("Person"), collections: ["people"] }, { key: "talk", type: "text", labels: L("Talk") }, + { key: "photo", type: "image", labels: L("Photo") }, ], }; const ROOM: FieldDefinition = { @@ -431,7 +437,140 @@ export function referencesSuite(ctx: SuiteContext): void { }); }); + describe("assets inside repeater items (§2.5)", () => { + const upload = async (siteId: string, bytes: Uint8Array, filename: string) => + ( + await store().ingestAsset({ + siteId, + bytes, + filename, + uploadedBy: person, + source: { kind: "upload" }, + fields: { decorative: true }, + }) + ).asset; + const assetRows = (nodeId: string) => + ctx + .sql() + .all<{ asset_id: string; field: string }>( + "SELECT asset_id, field FROM asset_refs WHERE node_id = ? AND field LIKE 'fields.%'", + [nodeId], + ); + + it("records an item's image, keeps it through a rebuild, and blocks its delete", async () => { + const site = await world(); + const photo = await upload(site.id, PNG_1X1, "dot.png"); + const event = await entry(site, "events", { + fields: { speakers: [{ talk: "Opening" }, { talk: "Talk", photo: photo.id }] }, + }); + expect(await assetRows(event.id)).toEqual([ + { asset_id: photo.id, field: "fields.speakers.photo" }, + ]); + expect(await code(store().deleteAsset(photo))).toBe("in_use"); + const bundle = await exportSite(storage(), site.id); + await update(await fresh(event), { title: "Moved on" }); + await importSite(storage(), bundle, { mode: "replace", fieldTypes: ENABLED }); + expect(await assetRows(event.id)).toHaveLength(1); + }); + + it("refuses a non-image in an image sub-field at its item path", async () => { + const site = await world(); + const csv = await upload(site.id, new TextEncoder().encode("a,b\n1,2\n"), "data.csv"); + try { + await entry(site, "events", { fields: { speakers: [{ talk: "x" }, { photo: csv.id }] } }); + throw new Error("expected a refusal"); + } catch (e) { + const issues = (e as InvariantError).details.issues as { path: string }[]; + expect(issues.map((i) => i.path)).toEqual(["fields/speakers/1/photo"]); + } + }); + + it("reports a bad asset at every item that uses it", async () => { + const site = await world(); + const csv = await upload(site.id, new TextEncoder().encode("a,b\n1,2\n"), "data.csv"); + const speakers = [{ photo: csv.id }, { talk: "x" }, { photo: csv.id }]; + try { + await entry(site, "events", { fields: { speakers } }); + throw new Error("expected a refusal"); + } catch (e) { + const issues = (e as InvariantError).details.issues as { path: string }[]; + expect(issues.map((i) => i.path)).toEqual([ + "fields/speakers/0/photo", + "fields/speakers/2/photo", + ]); + } + }); + }); + describe("field operations (§2.6)", () => { + const BLURB: FieldDefinition = { + key: "blurb", + type: "text", + labels: L("Blurb"), + localized: true, + }; + + it("a text field's rename rewrites entries holding references and keeps their rows", async () => { + const site = await world({ ...COLLECTIONS, events: { fields: [VENUE, BLURB] } }); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { + status: "published", + fields: { venue: hall.id, blurb: "Hi" }, + }); + const op = { + op: "rename_field", + collection: "events", + from: "blurb", + to: "teaser", + } as const; + await runFieldOperation(store(), site.id, op, { actor: person }); + expect((await fresh(event)).fields).toEqual({ venue: hall.id, teaser: "Hi" }); + expect(await rows(event.id)).toEqual([ + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "published", + }, + ]); + }); + + it("R9: a rewrite changing a reference value skips the entry when its target goes first", async () => { + // Operations that convert into references come with rollout step 4; the rewrite path they + // will use is driven directly here, with a decision that changes a reference value. + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const pass = async (on: SiteGraphStore, target: string) => { + const ctx2 = operationContext(on, {}); + return rewritePass({ + ctx: ctx2, + site: (await store().getSite(site.id)) as Site, + rows: await loadEntries(ctx2, site.id, "events"), + defs: [VENUE, SPONSOR, SPEAKERS], + decide: (row) => + row.entry.id === event.id ? { fields: { venue: target } } : undefined, + guard: undefined, + done: 0, + lists: { collection: "events", keys: ["venue"] }, + }); + }; + const gone = await entry(site, "places"); + const raced = racing(async () => { + await store().deleteContentEntry(await fresh(gone)); + }); + const r = await pass(raced, gone.id); + expect(r.rewritten).toBe(0); + expect(r.skipped.map((x) => x.entryId)).toEqual([event.id]); + expect((await fresh(event)).fields?.venue).toBe(hall.id); + expect((await rows(event.id)).map((x) => x.target_id)).toEqual([hall.id]); + // without the race the rewrite lands, rows and all + const kept = await entry(site, "places"); + expect((await pass(store(), kept.id)).rewritten).toBe(1); + expect((await rows(event.id)).map((x) => x.target_id)).toEqual([kept.id]); + }); + it("refuses operations that name a reference or repeater field (step 4)", async () => { const site = await world(); for (const op of [ @@ -444,6 +583,325 @@ export function referencesSuite(ctx: SuiteContext): void { }); }); + describe("language editions (§3)", () => { + it("carries ids the edition can resolve, drops the rest and emptied items, and guards the group there", async () => { + const site = await world(); + const edition = await store().createSite({ + projectId: site.projectId, + name: "Edition", + slug: `refs-edition-${n}`, + defaultLocale: "de", + locales: ["de"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: COLLECTIONS, + }); + const hall = await entry(site, "places", { status: "published" }); + const ada = await entry(site, "people"); + const hallDe = await entry(edition, "places", { + locale: "de", + translation: { + group: hall.id, + method: "human", + source: { ...(await sourceOf(hall)), siteId: site.id }, + }, + }); + const event = await entry(site, "events", { + fields: { + venue: hall.id, + speakers: [{ person: ada.id }, { person: ada.id, talk: "Hi" }], + }, + }); + let dropped: { path: string; groupId?: string }[] = []; + const copy = await forEdition(store(), event, { + fromSiteId: site.id, + toSiteId: edition.id, + actor: person, + onDropped: (d) => { + dropped = d; + }, + }); + expect(copy.fields).toEqual({ venue: hall.id, speakers: [{ talk: "Hi" }] }); + expect(dropped).toEqual([ + { path: "fields/speakers/0/person", groupId: ada.id, reason: "missing" }, + { path: "fields/speakers/0", reason: "empty" }, + { path: "fields/speakers/1/person", groupId: ada.id, reason: "missing" }, + ]); + // the edition's event names the source site's group id, resolved by the edition's member + await entry(edition, "events", { + locale: "de", + status: "published", + fields: { venue: hall.id }, + }); + expect(await code(store().deleteContentEntry(await fresh(hallDe)))).toBe( + "entry_referenced", + ); + // and the source site's original is free: the edition's rows pin nothing there + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + }); + it("a re-save keeps the values the stored edition entry already holds (not fresh)", async () => { + const OPEN: FieldDefinition = { + key: "open", + type: "boolean", + labels: L("Open"), + localized: false, + }; + const source = await world(); + const edition = await store().createSite({ + projectId: source.projectId, + name: "Edition", + slug: `refs-edition-resave-${n}`, + defaultLocale: "de", + locales: ["de"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: { + ...COLLECTIONS, + places: { fields: [OPEN], listFields: ["open"] }, + events: { + fields: [ + { + ...VENUE, + collections: ["places"], + where: [{ field: "open", op: "eq", value: "true" }], + } as never, + SPONSOR, + SPEAKERS, + ], + }, + }, + }); + const hall = await entry(source, "places", { status: "published" }); + const hallDe = await entry(edition, "places", { + locale: "de", + fields: { open: true }, + translation: { + group: hall.id, + method: "human", + source: { ...(await sourceOf(hall)), siteId: source.id }, + }, + }); + const event = await entry(source, "events", { fields: { venue: hall.id } }); + const stored = await entry(edition, "events", { locale: "de", fields: { venue: hall.id } }); + // the target closes and is archived after the edition entry chose it + await update(await fresh(hallDe), { fields: { open: false }, status: "archived" }); + let dropped: unknown[] = []; + const copy = await forEdition( + store(), + { ...event, id: stored.id }, + { + fromSiteId: source.id, + toSiteId: edition.id, + actor: person, + onDropped: (d) => { + dropped = d; + }, + }, + ); + expect(copy.fields).toEqual({ venue: hall.id }); + expect(dropped).toEqual([]); + // and the edition's write keeps it too: an unchanged value is not re-checked + expect( + await code(update(await fresh(stored), { title: "Wieder", fields: copy.fields })), + ).toBe("ok"); + }); + + it("counts every unchanged reference toward the cap first, repeated targets included", async () => { + const PERSON = { + key: "person", + type: "reference", + labels: L("Person"), + collections: ["people"], + }; + const crew = (key: string) => + ({ + key, + type: "repeater", + labels: L(key), + localized: false, + maxItems: 50, + fields: [PERSON], + }) as never; + const site = await world({ + ...COLLECTIONS, + events: { + fields: [ + { ...VENUE, multiple: true, maxItems: 5 } as never, + crew("speakers"), + crew("crew"), + ], + }, + }); + const hall = await entry(site, "places"); + const ada = await entry(site, "people"); + // 100 references to one group in two unchanged repeaters: the cap is full + const items = Array.from({ length: 50 }, () => ({ person: ada.id })); + const stored = await entry(site, "events", { fields: { speakers: items, crew: items } }); + const { fields, dropped } = await editionReferences( + ctx.sql(), + (await store().getSite(site.id)) as Site, + { + entry: { + collection: "events", + fields: { venue: [hall.id], speakers: items, crew: items }, + }, + stored, + }, + ); + // the changed key comes first in the definitions, yet the unchanged ones count first + expect(fields).toEqual({ speakers: items, crew: items }); + expect(dropped).toEqual([{ path: "fields/venue/0", groupId: hall.id, reason: "cap" }]); + expect(await code(update(await fresh(stored), { fields: fields ?? {} }))).toBe("ok"); + expect( + await code( + update(await fresh(stored), { + fields: { venue: [hall.id], speakers: items, crew: items }, + }), + ), + ).toBe("too_many_references"); + }); + + it("an archived same-site original still lends its where field to a translation member", async () => { + const OPEN: FieldDefinition = { + key: "open", + type: "boolean", + labels: L("Open"), + localized: false, + }; + const site = await world({ + ...COLLECTIONS, + places: { fields: [OPEN], listFields: ["open"] }, + events: { + fields: [ + { + ...VENUE, + collections: ["places"], + where: [{ field: "open", op: "eq", value: "true" }], + } as never, + SPONSOR, + SPEAKERS, + ], + }, + }); + const original = await entry(site, "places", { fields: { open: true } }); + await translate(original, { fields: {} }); + await update(await fresh(original), { status: "archived" }); + const { fields, dropped } = await editionReferences( + ctx.sql(), + (await store().getSite(site.id)) as Site, + { + entry: { collection: "events", fields: { venue: original.id } }, + }, + ); + expect(dropped).toEqual([]); + expect(fields).toEqual({ venue: original.id }); + }); + + it("keeps only what the edition's field accepts: where, a single target, maxItems, fromKey", async () => { + const OPEN: FieldDefinition = { + key: "open", + type: "boolean", + labels: L("Open"), + localized: false, + }; + const source = await world({ + ...COLLECTIONS, + events: { fields: [{ ...VENUE, multiple: true } as never, SPONSOR, SPEAKERS] }, + }); + const edition = await store().createSite({ + projectId: source.projectId, + name: "Edition", + slug: `refs-edition-rules-${n}`, + defaultLocale: "de", + locales: ["de"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: { + ...COLLECTIONS, + places: { fields: [OPEN], listFields: ["open"] }, + events: { + fields: [ + { + ...VENUE, + collections: ["places"], + where: [{ field: "open", op: "eq", value: "true" }], + } as never, + SPONSOR, + { ...SPEAKERS, maxItems: 1 } as never, + ], + }, + }, + }); + const member = async (open: boolean) => { + const original = await entry(source, "places", { status: "published" }); + await entry(edition, "places", { + locale: "de", + fields: { open }, + translation: { + group: original.id, + method: "human", + source: { ...(await sourceOf(original)), siteId: source.id }, + }, + }); + return original; + }; + const closed = await member(false); + const open1 = await member(true); + const open2 = await member(true); + const event = await entry(source, "events", { + fields: { + venue: [closed.id, open1.id, open2.id], + speakers: [{ talk: "One" }, { talk: "Two" }], + }, + }); + let dropped: { path: string; groupId?: string; reason: string }[] = []; + const copy = await forEdition(store(), event, { + fromSiteId: source.id, + toSiteId: edition.id, + actor: person, + onDropped: (d) => { + dropped = d; + }, + }); + expect(copy.fields).toEqual({ venue: open1.id, speakers: [{ talk: "One" }] }); + expect(dropped).toEqual([ + { path: "fields/venue/0", groupId: closed.id, reason: "where" }, + { path: "fields/venue/2", groupId: open2.id, reason: "single" }, + { path: "fields/speakers/1", reason: "maxItems" }, + ]); + // the copy is a write the edition accepts + expect(await code(entry(edition, "events", { locale: "de", fields: copy.fields }))).toBe( + "ok", + ); + // a running rename on the edition: values are read, filtered and kept under the old key + await rawSite(edition.id, (s) => + withFields(s, "events", [ + { + ...VENUE, + key: "place", + collections: ["places"], + migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, + } as never, + SPONSOR, + SPEAKERS, + ]), + ); + const renamed = await forEdition( + store(), + { ...event, fields: { venue: ["entry_01HZX8K3QGONE"] } }, + { + fromSiteId: source.id, + toSiteId: edition.id, + actor: person, + }, + ); + expect(renamed.fields).toEqual({}); + }); + }); + describe("review of PR #205", () => { /** Reverses every object's key order: a site row an older import stored verbatim. */ const verbatim = (value: unknown): unknown => { diff --git a/packages/core-ts/test/references-store.test.ts b/packages/core-ts/test/references-store.test.ts index 4ca41c05..27136dd6 100644 --- a/packages/core-ts/test/references-store.test.ts +++ b/packages/core-ts/test/references-store.test.ts @@ -12,12 +12,14 @@ import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; import { describe, expect, it } from "vitest"; import { ConflictError } from "../src/errors.ts"; import { canonicalJson } from "../src/hash.ts"; +import { fieldAssetIssues, fieldAssetRefs } from "../src/media/field-refs.ts"; import { REFS_STATEMENT_BYTES } from "../src/media/refs-rebuild.ts"; import { mapUniqueViolation } from "../src/node-sql.ts"; import { entryRefsRebuild } from "../src/references/rebuild.ts"; import { entryRefRows } from "../src/references/rows.ts"; import { collectionsAnchor, definitionsAnchor } from "../src/references/sql.ts"; import { entryRefStatements } from "../src/references/writer.ts"; +import { releaseDepsOf, stripReferences } from "../src/release-deps.ts"; const FIXTURES = join(import.meta.dirname, "../../core-contracts/fixtures/valid/site"); const L = (label: string) => ({ en: { label } }); @@ -238,3 +240,97 @@ describe("the rows INSERT reads shared from the site row at batch time", () => { await expect(run([])).rejects.toThrow(/NOT NULL constraint failed: entry_refs\.shared/); }); }); + +describe("release dependencies (§2.7)", () => { + const site = { + collections: { + events: { + fields: [ + { + key: "place", + type: "reference", + labels: L("Venue"), + localized: false, + collections: ["places"], + migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, + }, + { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["people"] }, + { key: "photo", type: "image", labels: L("Photo") }, + ], + }, + ], + }, + }, + } as unknown as Site; + const body = { + id: "entry_01HZX8K3QSELF", + collection: "events", + coverImage: { assetId: "asset_01HZX8K3QCOVR" }, + terms: { tag: { ids: ["term_01HZX8K3QTAG1"] } }, + fields: { + venue: "entry_01HZX8K3QVENU", + speakers: [{ person: "entry_01HZX8K3QPERS", photo: "asset_01HZX8K3QFACE" }], + }, + }; + + it("drops reference values, top level and in items, and keeps every other dependency", async () => { + const deps = releaseDepsOf(site, async () => JSON.stringify(body)); + expect(await deps({ id: body.id, kind: "contentEntry", rev: 1, hash: "h" })).toEqual([ + "asset_01HZX8K3QCOVR", + "asset_01HZX8K3QFACE", + "term_01HZX8K3QTAG1", + ]); + // other kinds are walked as before + expect(await deps({ id: "page_01HZX8K3QPAGE", kind: "page", rev: 1, hash: "h" })).toContain( + "entry_01HZX8K3QVENU", + ); + }); + + it("leaves bodies without reference definitions as they are", () => { + expect(stripReferences({ collections: {} }, body)).toBe(body); + }); +}); + +describe("asset issues inside repeater items (§2.5)", () => { + const speakers: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "talk", type: "text", labels: L("Talk") }, + { key: "photo", type: "image", labels: L("Photo") }, + ], + }; + const CSV = "asset_01HZX8K3QCSV1"; + const lookup = async () => new Map([[CSV, { id: CSV, kind: "file", status: "active" } as never]]); + const values = { speakers: [{ photo: CSV }, { talk: "x" }, { photo: CSV }] }; + + it("one row per asset and field, every use's path kept", () => { + expect(fieldAssetRefs([speakers], values)).toEqual([ + { + assetId: CSV, + field: "fields.speakers.photo", + path: "fields/speakers/0/photo", + paths: ["fields/speakers/0/photo", "fields/speakers/2/photo"], + }, + ]); + }); + + it("reports each use, and skips only the uses under a skip path", async () => { + const refs = fieldAssetRefs([speakers], values); + const paths = async (skip: string[]) => + ( + await fieldAssetIssues({ defs: [speakers], refs, kept: [], lookup, skip: new Set(skip) }) + ).map((i) => i.path); + expect(await paths([])).toEqual(["fields/speakers/0/photo", "fields/speakers/2/photo"]); + expect(await paths(["fields/speakers/0"])).toEqual(["fields/speakers/2/photo"]); + expect(await paths(["fields/speakers"])).toEqual([]); + }); +}); diff --git a/packages/mcp-server/src/entry-fields.ts b/packages/mcp-server/src/entry-fields.ts index aed43a97..8bea0d42 100644 --- a/packages/mcp-server/src/entry-fields.ts +++ b/packages/mcp-server/src/entry-fields.ts @@ -111,7 +111,8 @@ export async function checkEntryFields(ctx: ToolContext, check: FieldsCheck): Pr refs: fieldAssetRefs(defs, values), kept: stored ? fieldAssetRefs(fieldDefinitions(site, stored.collection), stored.fields) : [], lookup: (ids) => ctx.store.getAssets(site.id, ids), - skip: new Set(issues.map((i) => i.path.split("/").slice(0, 2).join("/"))), + // RFC-0045 §2.5: issue paths as they are, as the store's check builds them. + skip: new Set(issues.map((i) => i.path)), })), ); issues.push(...checkRequiredToPublish(site, { ...next, fields: values }, original)); diff --git a/packages/mcp-server/test/entry-fields-assets.test.ts b/packages/mcp-server/test/entry-fields-assets.test.ts new file mode 100644 index 00000000..3e6767b0 --- /dev/null +++ b/packages/mcp-server/test/entry-fields-assets.test.ts @@ -0,0 +1,60 @@ +/** + * RFC-0045 §2.5 over MCP: the precheck reports the same asset issues inside repeater items as the + * store, an item-level issue in one item not hiding a non-image in another (full issue paths, as + * `store/entry-fields.ts` builds its skip set). + */ +import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { InvariantError, SiteGraphStore, seedStarterSite } from "@cloudhal/core-ts"; +import { MemoryBlobStore, migrate, migrations } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { describe, expect, it } from "vitest"; +import type { ToolContext } from "../src/common.ts"; +import { checkEntryFields } from "../src/entry-fields.ts"; + +const L = (label: string) => ({ en: { label } }); +const SPEAKERS: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "talk", type: "text", labels: L("Talk"), maxLength: 10 }, + { key: "photo", type: "image", labels: L("Photo") }, + ], +}; + +describe("checkEntryFields: assets in repeater items", () => { + it("reports an item-level issue and another item's non-image together", async () => { + const sql = await BetterSqlite3Driver.open(":memory:"); + await migrate(sql, migrations); + const store = new SiteGraphStore({ sql, blobs: new MemoryBlobStore() }); + const seeded = await seedStarterSite(store, { name: "Events", slug: "events" }); + const csv = await store.ingestAsset({ + siteId: seeded.id, + bytes: new TextEncoder().encode("a,b\n1,2\n"), + filename: "data.csv", + uploadedBy: { kind: "human", id: "user_assets01", name: "Halil" }, + source: { kind: "upload" }, + }); + // The precheck takes the site as given: the definitions need no store capability here. + const site = { ...seeded, collections: { events: { fields: [SPEAKERS] } } } as Site; + const ctx = { store, principal: {} } as unknown as ToolContext; + const next = { + id: "entry_01HZX8K3QASSET", + siteId: site.id, + collection: "events", + status: "draft" as const, + fields: { speakers: [{ talk: "far too long a talk" }, { photo: csv.asset.id }] }, + }; + const error = await checkEntryFields(ctx, { site, next }).then( + () => undefined, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(InvariantError); + const issues = (error as InvariantError).details.issues as { path: string }[]; + expect(issues.map((i) => i.path).sort()).toEqual([ + "fields/speakers/0/talk", + "fields/speakers/1/photo", + ]); + }); +});