From 19e1bce44d753957a7f07fbc59543ff762fa682d Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:31:58 +0300 Subject: [PATCH 1/6] feat(core-ts): entry_refs, target guard and leave rules (RFC-0045 step 2a, #114) Migration 0033 adds entry_refs (in SITE_TABLES). Entry writes canonicalize changed member ids to group ids, AND the target guard and the definitions anchor into their CAS, and rewrite their rows with a plain INSERT that reads `shared` from the site row at batch time (a NULL fails the batch and maps to ConflictError). Deletes, moves and group changes carry the collections anchor and the blocking-rows condition with the planned pairs, and refuse during a conversion into references (field_operation_running). Translation writes carry the written-group and dependants guards (translation_group_in_use); readsOriginal/originalOf require an original's group to be its own id. Definition changes that would strand values are refused (field_key_in_use, removed target collections or sub-fields, multiple). Field operations naming the new types answer field_operation_unsupported. Imports store canonical bodies and rebuild entry_refs beside a written node set; the snapshot gate checks held site bodies of new snapshots; site rows stored verbatim are rewritten canonically (nodes.canonical job, local store open). SiteGraphStoreOptions.fieldTypes and CopySiteOptions.fieldTypes carry the capability; production passes none. Shared suite: testing-references.ts. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- packages/api/src/errors.ts | 8 + packages/cloudflare/src/canonical-rows-job.ts | 28 + packages/cloudflare/src/scheduler.ts | 2 + packages/core-ts/src/errors.ts | 29 +- .../core-ts/src/fields-operations/plan.ts | 19 +- .../core-ts/src/fields-operations/state.ts | 10 +- packages/core-ts/src/fields/refusal.ts | 1 + packages/core-ts/src/fields/resolve.ts | 13 +- packages/core-ts/src/fields/types.ts | 8 +- packages/core-ts/src/index.ts | 2 + packages/core-ts/src/local.ts | 3 + packages/core-ts/src/media/refs-rebuild.ts | 2 +- packages/core-ts/src/node-sql.ts | 7 + .../core-ts/src/references/canonical-rows.ts | 46 + packages/core-ts/src/references/capability.ts | 30 + .../src/references/definition-changes.ts | 155 ++++ packages/core-ts/src/references/leave.ts | 184 ++++ packages/core-ts/src/references/rebuild.ts | 74 ++ packages/core-ts/src/references/rows.ts | 91 ++ packages/core-ts/src/references/sql.ts | 108 +++ packages/core-ts/src/references/writer.ts | 289 ++++++ packages/core-ts/src/site-delete.ts | 1 + packages/core-ts/src/store.ts | 12 +- packages/core-ts/src/store/entries.ts | 41 +- packages/core-ts/src/store/entry-fields.ts | 2 + packages/core-ts/src/store/entry-integrity.ts | 46 + packages/core-ts/src/store/pages.ts | 15 +- packages/core-ts/src/store/site-fields.ts | 34 +- packages/core-ts/src/store/translations.ts | 73 +- packages/core-ts/src/store/types.ts | 7 + packages/core-ts/src/sync-bundle.ts | 6 + packages/core-ts/src/sync-plan.ts | 59 +- .../core-ts/src/testing-reference-gate.ts | 4 +- packages/core-ts/src/testing-references.ts | 851 ++++++++++++++++++ packages/core-ts/src/testing.ts | 2 + .../core-ts/test/references-store.test.ts | 187 ++++ packages/mcp-server/src/field-hints.ts | 14 + .../storage/migrations/0033_entry_refs.ts | 35 + packages/storage/migrations/index.ts | 2 + packages/storage/test/migrate.test.ts | 6 + 40 files changed, 2458 insertions(+), 48 deletions(-) create mode 100644 packages/cloudflare/src/canonical-rows-job.ts create mode 100644 packages/core-ts/src/references/canonical-rows.ts create mode 100644 packages/core-ts/src/references/capability.ts create mode 100644 packages/core-ts/src/references/definition-changes.ts create mode 100644 packages/core-ts/src/references/leave.ts create mode 100644 packages/core-ts/src/references/rebuild.ts create mode 100644 packages/core-ts/src/references/rows.ts create mode 100644 packages/core-ts/src/references/sql.ts create mode 100644 packages/core-ts/src/references/writer.ts create mode 100644 packages/core-ts/src/store/entry-integrity.ts create mode 100644 packages/core-ts/src/testing-references.ts create mode 100644 packages/core-ts/test/references-store.test.ts create mode 100644 packages/storage/migrations/0033_entry_refs.ts diff --git a/packages/api/src/errors.ts b/packages/api/src/errors.ts index 1e99617e..57e383a2 100644 --- a/packages/api/src/errors.ts +++ b/packages/api/src/errors.ts @@ -96,6 +96,14 @@ export const INVARIANT_STATUS: Record + new Date((Math.floor(after.getTime() / HOUR_MS) + 1) * HOUR_MS); + +export function canonicalRowsJob(sql: SqlDriver, log: (line: string) => void): SystemJobDefinition { + return { + name: "nodes.canonical", + cost: 1 + LIMIT, + next: everyHour, + async run() { + const r = await canonicalizeSiteRows(sql, { limit: LIMIT }); + if (r.rewritten > 0) log(`canonical rows: ${r.rewritten} site row(s) rewritten`); + }, + }; +} diff --git a/packages/cloudflare/src/scheduler.ts b/packages/cloudflare/src/scheduler.ts index c1b40ab0..a6b8a8d4 100644 --- a/packages/cloudflare/src/scheduler.ts +++ b/packages/cloudflare/src/scheduler.ts @@ -62,6 +62,7 @@ import { SqlScheduleStore } from "@cloudhal/schedules/sql"; import type { SqlDriver } from "@cloudhal/storage"; import { D1Driver } from "@cloudhal/storage/d1"; import { R2BlobStore } from "@cloudhal/storage/r2"; +import { canonicalRowsJob } from "./canonical-rows-job.ts"; import { flushCommentPurges, runCommentsHourly } from "./comments.ts"; import { listValuesJob } from "./list-values-job.ts"; import { PURGE_MAX_REQUESTS, PURGE_TAGS_PER_REQUEST } from "./purge.ts"; @@ -273,6 +274,7 @@ export function adminSystemJobs( }, listValuesJob(sql, { paid: env.CLOUDHAL_TICK_BUDGET === "paid", log }), oauthClientsPruneJob(sql, log), + canonicalRowsJob(sql, log), ]; if (o.commentsOn) jobs.push({ diff --git a/packages/core-ts/src/errors.ts b/packages/core-ts/src/errors.ts index 3d6b90fe..9a451c85 100644 --- a/packages/core-ts/src/errors.ts +++ b/packages/core-ts/src/errors.ts @@ -48,6 +48,19 @@ export function isTranslationViolation(error: unknown): boolean { return /nodes_translation_locale|json_extract\(body, '\$\.translation\.group'\)/i.test(msg); } +/** + * RFC-0045 §2.1: the `entry_refs` rows `INSERT` found no definition for a key at batch time (a + * concurrent definitions change). Matched by message only, so any other constraint failure still + * surfaces as itself. + */ +export function isSharedNullViolation(error: unknown): boolean { + const msg = + error instanceof Error + ? `${error.message} ${error.cause instanceof Error ? error.cause.message : ""}` + : String(error); + return /NOT NULL constraint failed: entry_refs\.shared/.test(msg); +} + export function isUniqueViolation(error: unknown): boolean { const msg = error instanceof Error @@ -217,7 +230,21 @@ export type InvariantCode = /** §1: a reference's `collections` names a collection the site neither declares nor uses. */ | "reference_target_unknown" /** §1: more than 100 reference values in one entry, across fields and repeater items. */ - | "too_many_references"; + | "too_many_references" + /** §2.1: a reference names a group with no usable member of this site in an allowed collection. */ + | "unknown_entry" + /** §2.1, §2.2: the group's member is in a collection the field does not allow, or a move would leave it so. */ + | "reference_target_moved" + /** §2.2: the delete or group change would strand a reference a published entry shows (`details.referrers`). */ + | "entry_referenced" + /** §1, §2.2: other nodes of the site use this node's id as their translation group. */ + | "translation_group_in_use" + /** §2.2: a field operation converting text into references is running in a target collection. */ + | "field_operation_running" + /** §2.6: field operations on reference and repeater fields come with rollout step 4. */ + | "field_operation_unsupported" + /** §2.8: adding a reference or repeater field over a key entries already hold. */ + | "field_key_in_use"; export class InvariantError extends Error { override readonly name = "InvariantError"; diff --git a/packages/core-ts/src/fields-operations/plan.ts b/packages/core-ts/src/fields-operations/plan.ts index 1bb508ab..eb52065d 100644 --- a/packages/core-ts/src/fields-operations/plan.ts +++ b/packages/core-ts/src/fields-operations/plan.ts @@ -15,7 +15,11 @@ const KEY = /^[a-z][A-Za-z0-9]{0,39}$/; const COLLECTION = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; function refuse( - code: "unknown_field" | "field_key_duplicate" | "field_key_reserved" | "fields_unsupported", + code: + | "unknown_field" + | "field_key_duplicate" + | "field_key_reserved" + | "field_operation_unsupported", msg: string, ) { return new InvariantError(code, msg, { issues: [] }); @@ -93,9 +97,16 @@ export function targetDefinition(def: FieldDefinition, op: FieldOperation): Fiel /** Checks the request against the collection; throws when it can never run as written. */ export function checkOperation(site: Site, op: FieldOperation): void { const { defs, def } = currentDefinition(site, op); - // RFC-0045: no operation turns a field into, or rewrites, a reference or repeater yet. - if (op.op === "change_field_type" && isReferenceFieldType(op.type)) - throw refuse("fields_unsupported", `${op.type} fields are not enabled on this server yet`); + // RFC-0045 §2.6: no operation turns a field into, or rewrites, a reference or repeater until + // rollout step 4. + if ( + isReferenceFieldType(def.type) || + (op.op === "change_field_type" && isReferenceFieldType(op.type)) + ) + throw refuse( + "field_operation_unsupported", + `field operations on reference and repeater fields are not available yet (${def.key})`, + ); if (op.op === "rename_field") { if (!KEY.test(op.to)) throw refuse("field_key_reserved", `${op.to} is not a field key (lower camelCase)`); diff --git a/packages/core-ts/src/fields-operations/state.ts b/packages/core-ts/src/fields-operations/state.ts index 122c4509..c1264ca6 100644 --- a/packages/core-ts/src/fields-operations/state.ts +++ b/packages/core-ts/src/fields-operations/state.ts @@ -5,6 +5,7 @@ * CAS itself, not `updateSite`: the plain-edit guard refuses markers by design, so only these * writes may set or drop one. They still pass the precompiled validator and the definition rules. */ + import { assertValid, type Site, type Snapshot } from "@cloudhal/core-contracts"; import type { SqlStatement } from "@cloudhal/storage"; import { ConflictError, InvariantError, NotFoundError } from "../errors.ts"; @@ -14,6 +15,7 @@ import { ID_PREFIX, newId } from "../ids.ts"; import { settleListValues } from "../list-values/backfill.ts"; import { planListState } from "../list-values/site-plan.ts"; import { type Cond, casUpdateStatement, OpBatch, runOpBatch } from "../node-sql.ts"; +import { referenceDefinitionOptions } from "../references/capability.ts"; import type { PageRewrite } from "./references.ts"; import { FIELD_MIGRATION, @@ -91,7 +93,13 @@ export async function writeSite( if (!changed) return "unchanged"; const next: Site = { ...changed, rev: site.rev + 1, updatedAt: ctx.stamp() }; assertValid("site", next); - refuseFieldIssues(checkFieldDefinitions(next)); + refuseFieldIssues( + checkFieldDefinitions( + next, + undefined, + await referenceDefinitionOptions(ctx.store, next, true), + ), + ); const rewrites = (await pages?.(site)) ?? []; // RFC-0042 §4: a marker on a listed field changes its fingerprint (phases A and C both). const lists = (await planListState(site, next)).statements( diff --git a/packages/core-ts/src/fields/refusal.ts b/packages/core-ts/src/fields/refusal.ts index f1d4f04d..1d0ff9f9 100644 --- a/packages/core-ts/src/fields/refusal.ts +++ b/packages/core-ts/src/fields/refusal.ts @@ -19,6 +19,7 @@ const DEFINITION_CODES = new Set([ "field_change_needs_operation", "fields_unsupported", "reference_target_unknown", + "field_key_in_use", ]); const asInvariant = (code: FieldIssueCode): InvariantCode => diff --git a/packages/core-ts/src/fields/resolve.ts b/packages/core-ts/src/fields/resolve.ts index c87c1726..82c9dbf3 100644 --- a/packages/core-ts/src/fields/resolve.ts +++ b/packages/core-ts/src/fields/resolve.ts @@ -41,13 +41,20 @@ export type ResolvedFields = Record; export type FieldsEntry = Pick & { translation?: ContentEntry["translation"] | undefined; }; -export type FieldsOriginal = Pick; +export type FieldsOriginal = Pick & { + translation?: Pick, "group"> | undefined; +}; -/** True when the entry is a translation whose original is in the same site. */ +/** + * True when the entry is a translation whose original is in the same site, and that original + * still has its own id as its group (RFC-0045 §2 "Membership", OQ 14: a relinked node inherits + * to nobody, so it never shows another group's values). + */ function readsOriginal(entry: FieldsEntry, original: FieldsOriginal | undefined): boolean { const t = entry.translation; if (!t || t.group === entry.id || t.source?.siteId !== undefined) return false; - return original !== undefined && original.id === t.group; + if (original === undefined || original.id !== t.group) return false; + return (original.translation?.group ?? original.id) === original.id; } function read(values: unknown, def: FieldDefinition): unknown { diff --git a/packages/core-ts/src/fields/types.ts b/packages/core-ts/src/fields/types.ts index 14a6fb03..2a4bbb62 100644 --- a/packages/core-ts/src/fields/types.ts +++ b/packages/core-ts/src/fields/types.ts @@ -151,7 +151,13 @@ export type FieldIssueCode = /** RFC-0045: a reference's `where` does not fit its target collection's list fields. */ | "invalid_query" /** RFC-0045: more than 100 reference values in one entry. */ - | "too_many_references"; + | "too_many_references" + /** RFC-0045 §2.1: a reference to a group without a usable member of this site. */ + | "unknown_entry" + /** RFC-0045 §2.1: the group's members are all in collections the field does not allow. */ + | "reference_target_moved" + /** RFC-0045 §2.8: a new reference or repeater field over a key entries already hold. */ + | "field_key_in_use"; /** One problem, at a slash path (`fields/startsAt/at`, `collections/events/fields/2/key`). */ export interface FieldIssue { diff --git a/packages/core-ts/src/index.ts b/packages/core-ts/src/index.ts index f4e9fe39..0fa2f947 100644 --- a/packages/core-ts/src/index.ts +++ b/packages/core-ts/src/index.ts @@ -79,6 +79,8 @@ export { redirectLocale, splitTarget, } from "./redirects.ts"; +// RFC-0045 §2.4: site rows in canonical form (the anchors compare them as text). +export { type CanonicalRowsResult, canonicalizeSiteRows } from "./references/canonical-rows.ts"; export type { PublishReleaseInput, PublishReleaseResult, diff --git a/packages/core-ts/src/local.ts b/packages/core-ts/src/local.ts index 7b5a770a..e7e4d7db 100644 --- a/packages/core-ts/src/local.ts +++ b/packages/core-ts/src/local.ts @@ -6,6 +6,7 @@ import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; import { GATES_ENFORCED } from "./availability.ts"; import type { BlockRegistry, PropsSchema } from "./blocks.ts"; import { rebuildListValues } from "./list-values/backfill.ts"; +import { canonicalizeSiteRows } from "./references/canonical-rows.ts"; import { SiteGraphStore, type SiteGraphStoreOptions } from "./store.ts"; export interface LocalStore { @@ -45,6 +46,8 @@ export async function openLocalStore( await migrate(sql, migrations); // RFC-0042 §4: list values for collections listed before migration 0029 or left unfinished. await rebuildListValues(sql); + // RFC-0045 §2.4: site rows stored verbatim by older imports, in canonical form for the anchors. + await canonicalizeSiteRows(sql); const blobs = new FsBlobStore(path.join(dir, "blobs")); // The renderer that serves this store runs in the same process and enforces gates (S15). const store = new SiteGraphStore({ diff --git a/packages/core-ts/src/media/refs-rebuild.ts b/packages/core-ts/src/media/refs-rebuild.ts index fd57d33b..78bb207f 100644 --- a/packages/core-ts/src/media/refs-rebuild.ts +++ b/packages/core-ts/src/media/refs-rebuild.ts @@ -28,7 +28,7 @@ const INSERT_SQL = `INSERT INTO asset_refs (site_id, asset_id, node_id, node_kin const encoder = new TextEncoder(); /** The JSON-encoded rows, in `json_each` arrays of at most `budget` bytes each. */ -function chunks(items: readonly string[], budget: number): string[] { +export function chunks(items: readonly string[], budget: number): string[] { const out: string[] = []; let open: string[] = []; let bytes = 2; diff --git a/packages/core-ts/src/node-sql.ts b/packages/core-ts/src/node-sql.ts index c0107486..44a5b3b6 100644 --- a/packages/core-ts/src/node-sql.ts +++ b/packages/core-ts/src/node-sql.ts @@ -7,6 +7,7 @@ import { ConflictError, InvariantError, isOpGuardViolation, + isSharedNullViolation, isTranslationViolation, isUniqueViolation, UniqueConstraintError, @@ -297,6 +298,12 @@ export function parseRows(rows: { body: string }[]): T[] { * took the locale in the group) or a node key; anything else is returned unchanged. */ export function mapUniqueViolation(e: unknown, kind: string, key: string): unknown { + // RFC-0045 §2.1: a reference key lost its definition between plan and batch; retry. + if (isSharedNullViolation(e)) { + const conflict = new ConflictError(kind, key, 0, undefined); + conflict.message = `${kind} ${key}: the collection's field definitions changed; reload and retry`; + return conflict; + } if (isTranslationViolation(e)) return new InvariantError( "translation_exists", diff --git a/packages/core-ts/src/references/canonical-rows.ts b/packages/core-ts/src/references/canonical-rows.ts new file mode 100644 index 00000000..1588ac63 --- /dev/null +++ b/packages/core-ts/src/references/canonical-rows.ts @@ -0,0 +1,46 @@ +/** + * RFC-0045 §2.4 "Canonical rows, from phase 2": the anchors compare a site row's `collections` + * and definitions as text, so a site row stored verbatim by an import before phase 2 (or by an + * import planned `unchanged`) could make every entry write of a collection conflict. This job + * rewrites such rows in canonical form, one CAS each (`WHERE hash = ?`, the hash unchanged: + * it is computed over canonical JSON already), so it can never race a write (rule 4). + * + * Only site rows are rewritten: they are what the anchors read. The admin Worker runs it as a + * system job, local stores when they open; step 7 runs it once more. Worker-safe. + */ +import type { SqlDriver } from "@cloudhal/storage"; +import { canonicalJson } from "../hash.ts"; + +export interface CanonicalRowsResult { + /** Site rows read. */ + read: number; + /** Rows rewritten (a row that changed meanwhile is skipped: its writer stored it canonically). */ + rewritten: number; +} + +/** Rewrites up to `limit` non-canonical site rows (default 50); idempotent. */ +export async function canonicalizeSiteRows( + sql: SqlDriver, + options: { limit?: number } = {}, +): Promise { + const rows = await sql.all<{ id: string; hash: string; body: string }>( + "SELECT id, hash, body FROM nodes WHERE kind = 'site' ORDER BY id", + ); + let rewritten = 0; + for (const row of rows) { + if (rewritten >= (options.limit ?? 50)) break; + let canonical: string; + try { + canonical = canonicalJson(JSON.parse(row.body)); + } catch { + continue; + } + if (canonical === row.body) continue; + const r = await sql.run( + "UPDATE nodes SET body = ? WHERE id = ? AND kind = 'site' AND hash = ? AND body = ?", + [canonical, row.id, row.hash, row.body], + ); + rewritten += r.changes; + } + return { read: rows.length, rewritten }; +} diff --git a/packages/core-ts/src/references/capability.ts b/packages/core-ts/src/references/capability.ts new file mode 100644 index 00000000..373cf2c6 --- /dev/null +++ b/packages/core-ts/src/references/capability.ts @@ -0,0 +1,30 @@ +/** + * RFC-0045 §1: the definition options of a site write from the store's capability + * (`SiteGraphStoreOptions.fieldTypes`): whether the types are accepted, and the inferred + * collections a reference may target. Absent capability: both types refused. + */ +import type { Site } from "@cloudhal/core-contracts"; +import type { CollectionInfo } from "../collections.ts"; +import { + type FieldTypeSupport, + type ReferenceDefinitionOptions, + referenceTypesSupported, +} from "./definitions.ts"; +import { hasReferenceFields } from "./issues.ts"; + +export interface CapableStore { + readonly fieldTypes: FieldTypeSupport | undefined; + listCollections(siteId: string): Promise; +} + +/** The options `checkFieldDefinitions` takes for a write of `site` (one read only when needed). */ +export async function referenceDefinitionOptions( + store: CapableStore, + site: Pick, + exists: boolean, +): Promise { + const referenceTypes = await referenceTypesSupported(site.id, store.fieldTypes); + if (!referenceTypes || !exists || !hasReferenceFields(site)) return { referenceTypes }; + const inUse = (await store.listCollections(site.id)).map((c) => c.key); + return { referenceTypes, inUse }; +} diff --git a/packages/core-ts/src/references/definition-changes.ts b/packages/core-ts/src/references/definition-changes.ts new file mode 100644 index 00000000..a15ca735 --- /dev/null +++ b/packages/core-ts/src/references/definition-changes.ts @@ -0,0 +1,155 @@ +/** + * RFC-0045 §1, §2.8: the definition changes of reference and repeater fields that depend on + * stored values, checked by a site write (`planFieldDefinitions`) and kept true by guards on the + * site CAS (rule 4); entry writes in between carry the definitions anchor (R3). + * + * - **Adding** a reference or repeater field over a key entries already hold (`field_key_in_use`). + * - **Removing a target collection** while a value's group has a member in it. + * - **Removing or retyping a sub-field** that items hold. + * - **Turning `multiple` on or off** is refused outright (`change_field_type`), as for selects. + * + * Each refusal names what to do instead. + */ +import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; +import type { FieldIssue } from "../fields/types.ts"; +import type { Cond } from "../node-sql.ts"; +import { memberSql } from "./sql.ts"; + +type Collections = NonNullable; + +/** One value-dependent change; `count` is the SQL counting what it would strand. */ +export interface ReferenceDataChange { + collection: string; + key: string; + issue(n: number): FieldIssue; + count: Cond; +} + +const IN_COLLECTION = + "site_id = ? AND kind = 'contentEntry' AND json_extract(body, '$.collection') = ?"; + +const entries = (n: number) => `${n} ${n === 1 ? "entry" : "entries"}`; + +function added(siteId: string, collection: string, def: FieldDefinition): ReferenceDataChange { + return { + collection, + key: def.key, + count: { + sql: `SELECT count(*) AS n FROM nodes WHERE ${IN_COLLECTION} AND json_type(body, ?) IS NOT NULL`, + params: [siteId, collection, `$.fields.${def.key}`], + }, + issue: (n) => ({ + path: `collections/${collection}/fields/${def.key}`, + code: "field_key_in_use", + message: `${entries(n)} of ${collection} already hold ${def.key}; choose another key, or use change_field_type`, + }), + }; +} + +function collectionsRemoved( + siteId: string, + at: { collection: string; key: string; field: string }, + removed: readonly string[], +): ReferenceDataChange { + return { + collection: at.collection, + key: at.key, + count: { + sql: `SELECT count(*) AS n FROM entry_refs r WHERE r.site_id = ? AND r.source_collection = ? AND r.field = ? + AND EXISTS (SELECT 1 FROM nodes n WHERE n.site_id = r.site_id AND n.kind = 'contentEntry' + AND ${memberSql("n", "r.target_id")} + AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(?)))`, + params: [siteId, at.collection, at.field, JSON.stringify(removed)], + }, + issue: (n) => ({ + path: `collections/${at.collection}/fields/${at.key}/collections`, + code: "field_change_needs_operation", + message: `${n} values of ${at.field} name entries in ${removed.join(", ")}; change them first`, + }), + }; +} + +function subFieldRemoved(siteId: string, collection: string, key: string, sub: string) { + return { + collection, + key, + count: { + sql: `SELECT count(*) AS n FROM nodes WHERE ${IN_COLLECTION} + AND EXISTS (SELECT 1 FROM json_each(body, ?) i WHERE json_type(i.value, ?) IS NOT NULL)`, + params: [siteId, collection, `$.fields.${key}`, `$.${sub}`], + }, + issue: (n: number): FieldIssue => ({ + path: `collections/${collection}/fields/${key}/fields`, + code: "field_change_needs_operation", + message: `${entries(n)} of ${collection} hold ${key}.${sub}; remove it with remove_field first`, + }), + } satisfies ReferenceDataChange; +} + +/** The changes of one field kept under its key (same type). */ +function compare( + siteId: string, + collection: string, + was: FieldDefinition, + now: FieldDefinition, +): { data: ReferenceDataChange[]; refused: FieldIssue[] } { + const data: ReferenceDataChange[] = []; + const refused: FieldIssue[] = []; + const at = `collections/${collection}/fields/${now.key}`; + if (was.type === "reference" && now.type === "reference") { + if ((was.multiple === true) !== (now.multiple === true)) + refused.push({ + path: at, + code: "field_change_needs_operation", + message: `changing multiple of ${now.key} needs change_field_type`, + }); + const gone = was.collections.filter((c) => !now.collections.includes(c)); + if (gone.length > 0) + data.push(collectionsRemoved(siteId, { collection, key: now.key, field: now.key }, gone)); + } + if (was.type === "repeater" && now.type === "repeater") + for (const sub of was.fields) { + const next = now.fields.find((s) => s.key === sub.key); + if (!next || next.type !== sub.type) { + data.push(subFieldRemoved(siteId, collection, now.key, sub.key)); + continue; + } + if (sub.type !== "reference" || next.type !== "reference") continue; + const gone = sub.collections.filter((c) => !next.collections.includes(c)); + const field = `${now.key}.${sub.key}`; + if (gone.length > 0) + data.push(collectionsRemoved(siteId, { collection, key: now.key, field }, gone)); + } + return { data, refused }; +} + +/** + * The reference and repeater changes from `before` to `after`: value-dependent ones to count and + * guard, and the ones always refused. A collection dropped entirely is RFC-0040's. + */ +export function referenceDefinitionChanges( + siteId: string, + before: { collections?: Site["collections"] | undefined } | undefined, + after: { collections?: Site["collections"] | undefined }, +): { data: ReferenceDataChange[]; refused: FieldIssue[] } { + const was: Collections = before?.collections ?? {}; + const now: Collections = after.collections ?? {}; + const data: ReferenceDataChange[] = []; + const refused: FieldIssue[] = []; + for (const [collection, settings] of Object.entries(now)) { + const old = new Map((was[collection]?.fields ?? []).map((d) => [d.key, d])); + for (const def of settings?.fields ?? []) { + if (def.type !== "reference" && def.type !== "repeater") continue; + const prior = old.get(def.key); + if (!prior) { + // A field an operation creates carries its marker; the operation owns its values. + if (!def.migration) data.push(added(siteId, collection, def)); + continue; + } + const c = compare(siteId, collection, prior, def); + data.push(...c.data); + refused.push(...c.refused); + } + } + return { data, refused }; +} diff --git a/packages/core-ts/src/references/leave.ts b/packages/core-ts/src/references/leave.ts new file mode 100644 index 00000000..128db6b1 --- /dev/null +++ b/packages/core-ts/src/references/leave.ts @@ -0,0 +1,184 @@ +/** + * RFC-0045 §2.2: writes that make a translation group lose a member in a site, a delete, a move + * to another collection, or a change of `translation.group`, and the conditions their batch + * carries so a group never loses its last usable member while a published entry shows it. + * + * - **The collections anchor**, always (R4, R5, R12): `site.collections` as planned. + * - **Blocking rows** ("shown by a published entry", `shownRows`) of the group, with the planned + * pairs P (`"."`): `NOT EXISTS (blocking) OR ( AND NOT EXISTS + * (blocking with a pair not in P))`, so a field the plan did not see forces a retry (R13). + * - **Usable members** are checked per pair: another member of the group, not archived, in a + * collection that pair's field allows. A move keeps the pairs whose field allows the new + * collection satisfied by the node itself, unless it is archived. + * - **`field_operation_running`**: while a marker converts text into references whose targets may + * sit in the node's collection, readers show groups with no rows yet, so these writes wait. + * + * Refusals: `entry_referenced` (delete, group change) and `reference_target_moved` (move), with up + * to 20 referrers and the total. + */ +import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { InvariantError } from "../errors.ts"; +import { fieldDefinitions } from "../fields/types.ts"; +import { and, type Cond } from "../node-sql.ts"; +import { hasReferenceFields } from "./issues.ts"; +import { memberGroup } from "./members.ts"; +import { definitionOfPath } from "./rows.ts"; +import { collectionsAnchor, otherMember, shownRows } from "./sql.ts"; + +/** What a delete, move or group change of an entry adds to its batch. */ +export interface LeavePlan { + readonly guard: Cond; + /** After a batch that matched no row: the refusal when the group is now locked. */ + diagnose(): Promise; +} + +/** The collections a running text → reference conversion may resolve targets in. */ +export function convertingTargets(site: Pick): Set { + const out = new Set(); + const into = (from: string | undefined) => from !== undefined && from !== "reference"; + for (const settings of Object.values(site.collections ?? {})) + for (const def of settings?.fields ?? []) { + const m = def.migration; + if (!m) continue; + if (def.type === "reference" && into(m.fromType)) for (const c of def.collections) out.add(c); + if (def.type === "repeater" && m.sub !== undefined) { + const sub = def.fields.find((s) => s.key === m.sub); + if (sub?.type === "reference" && into(m.fromType)) + for (const c of sub.collections) out.add(c); + } + } + return out; +} + +interface BlockingRow { + c: string; + f: string; + s: string; +} + +interface Pair { + key: string; + collections: readonly string[]; +} + +/** The pairs of the blocking rows, with each field's allowed collections from the planned site. */ +function pairsOf(site: Site, rows: readonly BlockingRow[]): Pair[] { + const out = new Map(); + for (const r of rows) { + const key = `${r.c}.${r.f}`; + if (out.has(key)) continue; + const defs: readonly FieldDefinition[] = fieldDefinitions(site, r.c); + out.set(key, { key, collections: definitionOfPath(defs, r.f)?.collections ?? [] }); + } + return [...out.values()]; +} + +/** Up to 20 referrers with their titles, for the refusal. */ +async function referrers(sql: SqlDriver, siteId: string, rows: readonly BlockingRow[]) { + const ids = [...new Set(rows.map((r) => r.s))]; + const titles = new Map(); + if (ids.length > 0) { + const found = await sql.all<{ id: string; title: string | null }>( + `SELECT id, json_extract(body, '$.title') AS title FROM nodes + WHERE site_id = ? AND id IN (SELECT value FROM json_each(?))`, + [siteId, JSON.stringify(ids.slice(0, 20))], + ); + for (const f of found) titles.set(f.id, f.title ?? ""); + } + const shown = rows.slice(0, 20).map((r) => ({ + id: r.s, + title: titles.get(r.s) ?? "", + collection: r.c, + field: r.f, + })); + return { referrers: shown, total: rows.length }; +} + +export interface LeaveInput { + /** The stored node. */ + stored: ContentEntry; + /** The node after the write; absent for a delete. */ + next?: ContentEntry | undefined; +} + +type Kind = "delete" | "group" | "move"; + +function kindOf({ stored, next }: LeaveInput): Kind | undefined { + if (!next) return "delete"; + if (memberGroup(next) !== memberGroup(stored)) return "group"; + return next.collection !== stored.collection ? "move" : undefined; +} + +function refuse(kind: Kind, group: string, details: Record): never { + const moved = kind === "move"; + throw new InvariantError( + moved ? "reference_target_moved" : "entry_referenced", + moved + ? `published entries show ${group} from a collection this move leaves; unpublish or edit them first` + : `published entries show ${group}, and this is its last usable entry in the site; unpublish or edit them first`, + { group, ...details }, + ); +} + +/** + * The plan of a delete, move or group change of an entry (undefined for other writes). Throws the + * plan-time refusals; the guard re-checks them at batch time. + */ +export async function planLeave( + sql: SqlDriver, + site: Site, + input: LeaveInput, +): Promise { + const kind = kindOf(input); + if (!kind) return undefined; + const { stored, next } = input; + const anchor = collectionsAnchor(site); + if (convertingTargets(site).has(stored.collection)) + throw new InvariantError( + "field_operation_running", + `a field operation is converting values into references to ${stored.collection}; wait until it finishes`, + { collection: stored.collection }, + ); + if (!hasReferenceFields(site)) return { guard: anchor, diagnose: async () => {} }; + const group = memberGroup(stored); + const shown = shownRows(site.id, group); + const read = () => + sql.all>( + `SELECT r.source_collection AS c, r.field AS f, r.source_id AS s FROM ${shown.sql}`, + shown.params, + ); + const rows = await read(); + const pairs = pairsOf(site, rows); + // A move keeps the pairs that allow its new collection (unless the node is archived). + const needs = pairs.filter( + (p) => + kind !== "move" || + next?.status === "archived" || + !p.collections.includes(next?.collection ?? ""), + ); + const members = needs.map((p) => otherMember(site.id, group, stored.id, p.collections)); + const check = async () => { + for (const m of members) { + const ok = await sql.get<{ ok: number }>(`SELECT ${m.sql} AS ok`, m.params); + if (!ok?.ok) refuse(kind, group, await referrers(sql, site.id, await read())); + } + }; + await check(); + const unseen: Cond = { + sql: `NOT EXISTS (SELECT 1 FROM ${shown.sql} AND r.source_collection || '.' || r.field NOT IN (SELECT value FROM json_each(?)))`, + params: [...shown.params, JSON.stringify(pairs.map((p) => p.key))], + }; + const none: Cond = { sql: `NOT EXISTS (SELECT 1 FROM ${shown.sql})`, params: shown.params }; + const kept = and(...members, unseen) as Cond; + return { + guard: and(anchor, { + sql: `${none.sql} OR (${kept.sql})`, + params: [...none.params, ...kept.params], + }) as Cond, + // The pairs may have grown since: planning again reads them and throws the refusal. + diagnose: async () => { + await planLeave(sql, site, input); + }, + }; +} diff --git a/packages/core-ts/src/references/rebuild.ts b/packages/core-ts/src/references/rebuild.ts new file mode 100644 index 00000000..764acdcc --- /dev/null +++ b/packages/core-ts/src/references/rebuild.ts @@ -0,0 +1,74 @@ +/** + * RFC-0045 §2.4: `entry_refs` for a whole site from its node rows, in the batch that writes the + * node set (`importSite`, so pull, push, `cloudhal import --replace` and `--revert`, and snapshot + * restores). Emitted only beside a written node set: an `unchanged` import's plan-time rows could + * replace a concurrent writer's fresher ones (R10, the `asset_refs` lesson of #196). + * + * `shared` is computed in JS from the same site row, so no `NULL` can arise, and body keys are + * read through any `migration.fromKey` the site carries (`entryRefRows`). There is no membership + * join: rows are kept for every stored value, and the site filters keep them harmless. The + * `INSERT` is chunked below 90 KB a statement; the chunks count against the import batch's cap. + * Worker-safe. + */ +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import { chunks, REFS_STATEMENT_BYTES } from "../media/refs-rebuild.ts"; +import { entryRefRowsOf } from "./rows.ts"; + +const INSERT_SQL = `INSERT INTO entry_refs (site_id, target_id, source_id, source_collection, field, root_key, shared, source_status) + SELECT ?, json_extract(r.value, '$.t'), json_extract(r.value, '$.n'), json_extract(r.value, '$.c'), + json_extract(r.value, '$.f'), json_extract(r.value, '$.k'), json_extract(r.value, '$.h'), json_extract(r.value, '$.s') + FROM json_each(?) r`; + +const encoder = new TextEncoder(); + +/** The rows of a site's entry bodies, JSON-encoded for `json_each`. */ +export function entryRefItems( + siteId: string, + rows: readonly { id: string; kind: string; body: string }[], +): string[] { + const siteRow = rows.find((r) => r.kind === "site" && r.id === siteId); + if (!siteRow) return []; + const site = JSON.parse(siteRow.body) as Pick; + const out: string[] = []; + for (const r of rows) { + if (r.kind !== "contentEntry") continue; + const entry = JSON.parse(r.body) as ContentEntry; + for (const row of entryRefRowsOf(site, entry)) + out.push( + JSON.stringify({ + t: row.t, + n: r.id, + c: entry.collection, + f: row.f, + k: row.k, + h: row.shared, + s: entry.status, + }), + ); + } + return out; +} + +/** `entry_refs` for a whole site from node rows: one `DELETE`, then `INSERT`s below 90 KB each. */ +export function entryRefsRebuild( + siteId: string, + rows: readonly { id: string; kind: string; body: string }[], +): SqlStatement[] { + const fixed = encoder.encode(INSERT_SQL + siteId).length; + return [ + { sql: "DELETE FROM entry_refs WHERE site_id = ?", params: [siteId] }, + ...chunks(entryRefItems(siteId, rows), REFS_STATEMENT_BYTES - fixed).map((items) => ({ + sql: INSERT_SQL, + params: [siteId, items], + })), + ]; +} + +/** A deleted node's rows, only once the node is gone (anchored like `dropRefsStatement`). */ +export function dropEntryRefsStatement(nodeId: string): SqlStatement { + return { + sql: "DELETE FROM entry_refs WHERE source_id = ? AND NOT EXISTS (SELECT 1 FROM nodes WHERE id = ?)", + params: [nodeId, nodeId], + }; +} diff --git a/packages/core-ts/src/references/rows.ts b/packages/core-ts/src/references/rows.ts new file mode 100644 index 00000000..47580554 --- /dev/null +++ b/packages/core-ts/src/references/rows.ts @@ -0,0 +1,91 @@ +/** + * RFC-0045 §2 "The index": what `entry_refs` holds for one stored body. One row per (node, field + * path, stored group id), whatever the target's state; `field` is `venue` or `speakers.person`, + * `root_key` the top-level key as stored (a running `rename_field` keeps the old key until the + * entry is rewritten, so keys are read through `migration.fromKey`, as `resolveFields` does). + * Pairs are deduplicated after the writer canonicalized member ids to group ids. Pure. + */ +import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { fieldDefinitions, isRecord } from "../fields/types.ts"; +import { referenceUses } from "./values.ts"; + +/** One row's varying part: target group, field path, top-level key. */ +export interface RefPair { + /** The stored translation group id. */ + t: string; + /** `venue`, or `speakers.person` inside a repeater. */ + f: string; + /** The top-level key the value is stored under. */ + k: string; +} + +/** A row as the rebuild writes it (`shared` computed from the definition). */ +export interface EntryRefRow extends RefPair { + shared: 1 | 2; +} + +/** `shared`: 1 for `localized: false` (every inheriting member shows it), 2 for `localized: true`. */ +export const sharedOf = (def: Pick): 1 | 2 => (def.localized ? 2 : 1); + +/** The JSON path of a collection's definitions in the site body (keys are slugs). */ +export const definitionsPath = (collection: string): string => + `$.collections."${collection}".fields`; + +/** The key `def`'s value is stored under in `values`: its own, else a running rename's old one. */ +function storedKey(def: FieldDefinition, values: Record): string | undefined { + if (Object.hasOwn(values, def.key)) return def.key; + const from = def.migration?.fromKey; + return from !== undefined && Object.hasOwn(values, from) ? from : undefined; +} + +/** True for definitions whose values can hold group ids. */ +export const holdsReferences = (def: FieldDefinition): boolean => + def.type === "reference" || + (def.type === "repeater" && def.fields.some((s) => s.type === "reference")); + +/** + * The rows of one body's values under `defs`, in order, each (field, target) once. A value that is + * not a reference value (a malformed import) yields nothing; readers treat it as unresolved. + */ +export function entryRefRows(defs: readonly FieldDefinition[], values: unknown): EntryRefRow[] { + if (!isRecord(values)) return []; + const out: EntryRefRow[] = []; + const seen = new Set(); + for (const def of defs) { + if (!holdsReferences(def)) continue; + const k = storedKey(def, values); + if (k === undefined) continue; + for (const use of referenceUses(def, values[k], "")) { + // "/1/person" → "person"; "" or "/2" → the field itself. + const sub = def.type === "repeater" ? use.path.split("/")[2] : undefined; + const f = sub === undefined ? k : `${k}.${sub}`; + const id = `${f}\u0000${use.groupId}`; + if (seen.has(id)) continue; + seen.add(id); + out.push({ t: use.groupId, f, k, shared: sharedOf(def) }); + } + } + return out; +} + +/** The rows of an entry body under its site's definitions. */ +export function entryRefRowsOf( + site: Pick, + entry: { collection: string; fields?: unknown }, +): EntryRefRow[] { + return entryRefRows(fieldDefinitions(site, entry.collection), entry.fields); +} + +/** The definition (or sub-field) a row's field path names, by key or a running rename's old key. */ +export function definitionOfPath( + defs: readonly FieldDefinition[], + field: string, +): { def: FieldDefinition; collections: readonly string[] } | undefined { + const [root, sub] = field.split("."); + const def = defs.find((d) => d.key === root) ?? defs.find((d) => d.migration?.fromKey === root); + if (!def) return undefined; + if (def.type === "reference" && sub === undefined) return { def, collections: def.collections }; + if (def.type !== "repeater" || sub === undefined) return undefined; + const s = def.fields.find((x) => x.key === sub); + return s?.type === "reference" ? { def, collections: s.collections } : undefined; +} diff --git a/packages/core-ts/src/references/sql.ts b/packages/core-ts/src/references/sql.ts new file mode 100644 index 00000000..101ae212 --- /dev/null +++ b/packages/core-ts/src/references/sql.ts @@ -0,0 +1,108 @@ +/** + * RFC-0045 §2: the batch-time SQL every reference guard shares, so the guards, the live + * `referencedBy` and Studio's "Used by" mean the same as `resolveFields`: + * + * - **membership** `COALESCE(translation.group, id) = G`, written so the primary key and + * `nodes_translation_locale` still serve it; + * - **"shown by a published entry"**: rows of site S naming G whose holder is published, or that a + * published same-site member inherits (`shared` 1, or 2 without its own value of `root_key`); + * - the **collections anchor** and the **definitions anchor**, bound from the `Site` object the + * write planned with (never a second read) and compared with `json(…) IS json(?)` over + * canonical bodies; an absent value binds SQL `NULL`, never the string `'null'`. + * + * Every probe filters `site_id`. Worker-safe; internal to core-ts. + */ +import type { Site } from "@cloudhal/core-contracts"; +import type { SqlValue } from "@cloudhal/storage"; +import { canonicalJson } from "../hash.ts"; +import type { Cond } from "../node-sql.ts"; +import { definitionsPath } from "./rows.ts"; + +/** `alias` is a member of the group the SQL expression `group` names (two uses of it). */ +export const memberSql = (alias: string, group: string): string => + `((${alias}.id = ${group} AND json_extract(${alias}.body, '$.translation.group') IS NULL) OR json_extract(${alias}.body, '$.translation.group') = ${group})`; + +/** The published-entry condition over `entry_refs r`, after `r.site_id = ? AND r.target_id = ?`. */ +const SHOWN = `(r.source_status = 'published' OR EXISTS ( + SELECT 1 FROM nodes m WHERE m.site_id = r.site_id AND m.kind = 'contentEntry' + AND json_extract(m.body, '$.translation.group') = r.source_id AND m.id <> r.source_id + AND json_extract(m.body, '$.translation.source.siteId') IS NULL + AND json_extract(m.body, '$.status') = 'published' + AND (r.shared = 1 OR json_type(m.body, '$.fields.' || r.root_key) IS NULL)))`; + +/** `FROM entry_refs r WHERE …`: the rows of site `siteId` naming `group` that a published entry shows. */ +export function shownRows(siteId: string, group: string): Cond { + return { + sql: `entry_refs r WHERE r.site_id = ? AND r.target_id = ? AND ${SHOWN}`, + params: [siteId, group], + }; +} + +/** Bound canonical JSON, or SQL `NULL` for an absent value (`json(NULL) IS json(NULL)` holds). */ +const bound = (value: unknown): SqlValue => (value === undefined ? null : canonicalJson(value)); + +/** + * §2.2: the site's `collections` are still those the write planned with. Every delete, move and + * group change of an entry carries it: it holds every definition and field-operation marker, and + * ignores `publishedSnapshotId`, theme, menus and other settings. + */ +export function collectionsAnchor(site: Pick): Cond { + return { + sql: "json((SELECT json_extract(body, '$.collections') FROM nodes WHERE id = ? AND kind = 'site')) IS json(?)", + params: [site.id, bound(site.collections)], + }; +} + +/** §2.1: one collection's definitions are still those the entry write planned with. */ +export function definitionsAnchor( + site: Pick, + collection: string, +): Cond { + return { + sql: "json((SELECT json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site')) IS json(?)", + params: [definitionsPath(collection), site.id, bound(site.collections?.[collection]?.fields)], + }; +} + +/** One target-guard pair: group, allowed collections, and whether the value is newly chosen. */ +export interface TargetPair { + t: string; + c: readonly string[]; + /** 1: newly chosen, so its member must not be archived. */ + f: 0 | 1; +} + +/** + * §2.1's target guard: every pair's group has a member in this site, in one of its allowed + * collections, and not archived when newly chosen. One count over one JSON parameter. + */ +export function targetGuard(siteId: string, pairs: readonly TargetPair[]): Cond | undefined { + if (pairs.length === 0) return undefined; + return { + sql: `(SELECT count(*) FROM json_each(?) p WHERE EXISTS ( + SELECT 1 FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' + AND ${memberSql("n", "json_extract(p.value, '$.t')")} + AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(json_extract(p.value, '$.c'))) + AND (json_extract(p.value, '$.f') = 0 OR json_extract(n.body, '$.status') <> 'archived'))) = ?`, + params: [JSON.stringify(pairs), siteId, pairs.length], + }; +} + +/** + * Another member of `group` than `nodeId` in this site, not archived, in one of `collections` + * (the delete, move and group-change rules' "usable member"). + */ +export function otherMember( + siteId: string, + group: string, + nodeId: string, + collections: readonly string[], +): Cond { + return { + sql: `EXISTS (SELECT 1 FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' AND n.id <> ? + AND ${memberSql("n", "?")} + AND json_extract(n.body, '$.status') <> 'archived' + AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(?)))`, + params: [siteId, nodeId, group, group, JSON.stringify(collections)], + }; +} diff --git a/packages/core-ts/src/references/writer.ts b/packages/core-ts/src/references/writer.ts new file mode 100644 index 00000000..5ee22b57 --- /dev/null +++ b/packages/core-ts/src/references/writer.ts @@ -0,0 +1,289 @@ +/** + * RFC-0045 §2.1: the reference part of an entry write (`EntryEngine.write`). + * + * - **Canonical values.** A member's id is replaced by its group id, `COALESCE(translation.group, + * id)`, before any check (as `canonicalImageRefs` does for assets). + * - **The target guard**, ANDed into the entry's CAS: for changed reference keys, and for every + * own reference value when the entry becomes published, each group has a member in this site in + * an allowed collection, not archived when newly chosen. A newly chosen target also meets + * `where`, and no value names the entry's own group (checked when planned). + * - **The definitions anchor**, on every write that changes `fields` or `collection`: the + * collection's definitions as planned. + * - **The rows**, anchored on the entry's new hash: an anchored `DELETE`, then one plain + * `INSERT … SELECT` that reads `shared` from the site row as it is when the batch runs, by key + * or `migration.fromKey`. A key with no definition then yields `NULL` and fails the batch + * (`isSharedNullViolation` → `ConflictError`); it is never `OR IGNORE`. + */ +import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; +import type { SqlDriver, SqlStatement } from "@cloudhal/storage"; +import { matchesFieldQuery, parseFieldQuery } from "../fields/query.ts"; +import { refuseFieldIssues } from "../fields/refusal.ts"; +import { resolveFields } from "../fields/resolve.ts"; +import { type FieldIssue, fieldDefinitions, isRecord, sameJson } from "../fields/types.ts"; +import { and, type Cond } from "../node-sql.ts"; +import { hasReferenceFields } from "./issues.ts"; +import { memberGroup } from "./members.ts"; +import { definitionsPath, entryRefRows, holdsReferences } from "./rows.ts"; +import { definitionsAnchor, type TargetPair, targetGuard } from "./sql.ts"; +import { type ReferenceUse, referenceUses } from "./values.ts"; + +const ANCHOR = "EXISTS (SELECT 1 FROM nodes WHERE id = ? AND hash = ?)"; + +interface MemberRow { + id: string; + group: string; + collection: string; + status: string; + body: ContentEntry; +} + +/** The members of `groups` (and the nodes named by id) in this site, one read. */ +async function readMembers( + sql: SqlDriver, + siteId: string, + ids: readonly string[], +): Promise { + if (ids.length === 0) return []; + const rows = await sql.all<{ id: string; body: string }>( + `SELECT n.id AS id, n.body AS body FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' + AND (n.id IN (SELECT value FROM json_each(?)) + OR json_extract(n.body, '$.translation.group') IN (SELECT value FROM json_each(?)))`, + [siteId, JSON.stringify(ids), JSON.stringify(ids)], + ); + return rows.map((r) => { + const body = JSON.parse(r.body) as ContentEntry; + return { + id: r.id, + group: memberGroup(body), + collection: body.collection, + status: body.status, + body, + }; + }); +} + +/** Every group id a body's values name under `defs`. */ +const namedIds = (defs: readonly FieldDefinition[], values: unknown): string[] => [ + ...new Set(entryRefRows(defs, values).map((r) => r.t)), +]; + +/** Replaces the reference values of one field with `map(id)`. */ +function mapValue(def: FieldDefinition, value: unknown, map: (id: string) => string): unknown { + if (def.type === "reference") { + if (typeof value === "string") return map(value); + return Array.isArray(value) + ? [...new Set(value.map((v) => (typeof v === "string" ? map(v) : v)))] + : value; + } + if (def.type !== "repeater" || !Array.isArray(value)) return value; + const subs = def.fields.filter((s) => s.type === "reference").map((s) => s.key); + return value.map((item) => { + if (!isRecord(item)) return item; + const next = { ...item }; + for (const k of subs) + if (Object.hasOwn(next, k) && typeof next[k] === "string") next[k] = map(next[k] as string); + return next; + }); +} + +/** + * The draft with every member id in a *changed* value canonicalized to its group id (one read, + * only when such a value names ids). A stored value that does not change is kept as stored: its + * id names the group it named when written, even if that node has since joined another group. + */ +export async function canonicalReferences( + sql: SqlDriver, + site: Site, + input: { draft: ContentEntry; stored?: ContentEntry | undefined }, +): Promise { + const { draft, stored } = input; + const before = stored?.collection === draft.collection ? (stored.fields ?? {}) : {}; + const values = draft.fields ?? {}; + const defs = fieldDefinitions(site, draft.collection).filter( + (d) => + holdsReferences(d) && Object.hasOwn(values, d.key) && !sameJson(values[d.key], before[d.key]), + ); + const ids = namedIds(defs, values); + if (ids.length === 0) return draft; + const rows = await sql.all<{ id: string; g: string | null }>( + `SELECT id, json_extract(body, '$.translation.group') AS g FROM nodes + WHERE site_id = ? AND kind = 'contentEntry' AND id IN (SELECT value FROM json_each(?))`, + [site.id, JSON.stringify(ids)], + ); + const groups = new Map(rows.flatMap((r) => (r.g && r.g !== r.id ? [[r.id, r.g] as const] : []))); + if (groups.size === 0) return draft; + const fields = { ...values }; + for (const def of defs) + fields[def.key] = mapValue(def, fields[def.key], (id) => groups.get(id) ?? id); + return { ...draft, fields }; +} + +/** What the reference rules add to an entry write. */ +export interface EntryRefsPlan { + readonly guard: Cond | undefined; + /** The `entry_refs` rows, anchored on the entry's new hash. */ + statements(hash: string): SqlStatement[]; + /** After a write that matched no row: `unknown_entry` / `reference_target_moved` if a target went. */ + diagnose(): Promise; +} + +interface Use extends ReferenceUse { + fresh: boolean; + def: FieldDefinition; +} + +/** The uses the target guard covers: changed keys, or every own value when the entry goes live. */ +function guardedUses( + defs: readonly FieldDefinition[], + draft: ContentEntry, + stored: ContentEntry | undefined, +): Use[] { + const values = draft.fields ?? {}; + const same = stored?.collection === draft.collection; + const before = same ? (stored?.fields ?? {}) : {}; + const goingLive = draft.status === "published" && (!same || stored?.status !== "published"); + const out: Use[] = []; + for (const def of defs) { + if (!holdsReferences(def) || !Object.hasOwn(values, def.key)) continue; + const changed = !sameJson(values[def.key], before[def.key]); + if (!changed && !goingLive) continue; + const had = new Set(referenceUses(def, before[def.key], "").map((u) => u.groupId)); + for (const use of referenceUses(def, values[def.key], `fields/${def.key}`)) + out.push({ ...use, def, fresh: !had.has(use.groupId) }); + } + return out; +} + +/** Why `use` fails against the members read; undefined when a member satisfies it. */ +function useIssue(site: Site, use: Use, members: readonly MemberRow[]): FieldIssue | undefined { + const of = members.filter((m) => m.group === use.groupId); + const allowed = of.filter((m) => use.collections.includes(m.collection)); + const usable = allowed.filter((m) => !use.fresh || m.status !== "archived"); + if (usable.length === 0) { + if (of.length > 0 && allowed.length === 0) + return { + path: use.path, + code: "reference_target_moved", + message: `${use.groupId} has no entry in ${use.collections.join(", ")} of this site`, + }; + return { + path: use.path, + code: "unknown_entry", + message: + allowed.length > 0 + ? `${use.groupId} is archived; choose an entry that is not` + : `${use.groupId} is not an entry of this site`, + }; + } + const where = use.def.type === "reference" && use.fresh ? use.def.where : undefined; + if (!where) return undefined; + const target = use.collections[0] as string; + const parsed = parseFieldQuery(site, target, { where }); + if (!parsed.ok) return undefined; + const byId = new Map(members.map((m) => [m.id, m.body])); + const meets = usable.some((m) => { + const original = byId.get(m.group); + const resolved = resolveFields(site, m.body, original); + // `where` cannot use `now` (§1), so the instant is irrelevant. + return matchesFieldQuery(resolved, parsed.query, 0); + }); + return meets + ? undefined + : { + path: use.path, + code: "invalid_value", + message: `${use.groupId} does not meet the conditions of ${use.def.key}`, + }; +} + +/** The plan-time checks of the guarded uses: own group, targets, `where`. */ +function useIssues(site: Site, draft: ContentEntry, uses: readonly Use[], members: MemberRow[]) { + const own = memberGroup(draft); + const issues: FieldIssue[] = []; + for (const use of uses) { + if (use.groupId === own) { + issues.push({ + path: use.path, + code: "invalid_value", + message: "an entry cannot reference its own translation group", + }); + continue; + } + const issue = useIssue(site, use, members); + if (issue) issues.push(issue); + } + return issues; +} + +/** The rows statements for `draft` (an anchored `DELETE`, then the plain `INSERT … SELECT`). */ +function rowStatements(site: Site, draft: ContentEntry, hash: string): SqlStatement[] { + const rows = entryRefRows(fieldDefinitions(site, draft.collection), draft.fields); + const out: SqlStatement[] = [ + { + sql: `DELETE FROM entry_refs WHERE source_id = ? AND ${ANCHOR}`, + params: [draft.id, draft.id, hash], + }, + ]; + if (rows.length === 0) return out; + out.push({ + sql: `INSERT INTO entry_refs (site_id, target_id, source_id, source_collection, field, root_key, shared, source_status) + SELECT ?, json_extract(p.value, '$.t'), ?, ?, json_extract(p.value, '$.f'), json_extract(p.value, '$.k'), + (SELECT CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END + FROM nodes s, json_each(json_extract(s.body, ?)) d + WHERE s.id = ? AND s.kind = 'site' + AND (json_extract(d.value, '$.key') = json_extract(p.value, '$.k') + OR json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k')) + ORDER BY json_extract(d.value, '$.key') = json_extract(p.value, '$.k') DESC LIMIT 1), + ? + FROM json_each(?) p WHERE ${ANCHOR}`, + params: [ + site.id, + draft.id, + draft.collection, + definitionsPath(draft.collection), + site.id, + draft.status, + JSON.stringify(rows.map(({ t, f, k }) => ({ t, f, k }))), + draft.id, + hash, + ], + }); + return out; +} + +/** True when the write changes the entry's values or collection (the definitions anchor applies). */ +const changesFields = (draft: ContentEntry, stored: ContentEntry | undefined) => + !stored || stored.collection !== draft.collection || !sameJson(stored.fields, draft.fields); + +/** + * Plans the reference part of an entry write; throws the plan-time refusals. Sites without + * reference fields add only the definitions anchor (to keep `field_key_in_use` true, R3). + */ +export async function planEntryRefs( + sql: SqlDriver, + site: Site, + draft: ContentEntry, + stored: ContentEntry | undefined, +): Promise { + const anchor = changesFields(draft, stored) + ? definitionsAnchor(site, draft.collection) + : undefined; + if (!hasReferenceFields(site)) + return { guard: anchor, statements: () => [], diagnose: async () => {} }; + const defs = fieldDefinitions(site, draft.collection); + const uses = guardedUses(defs, draft, stored); + const groups = [...new Set(uses.map((u) => u.groupId))]; + const check = async () => + refuseFieldIssues(useIssues(site, draft, uses, await readMembers(sql, site.id, groups))); + await check(); + const pairs = new Map(); + for (const u of uses) { + const pair: TargetPair = { t: u.groupId, c: u.collections, f: u.fresh ? 1 : 0 }; + pairs.set(JSON.stringify(pair), pair); + } + return { + guard: and(anchor, targetGuard(site.id, [...pairs.values()])), + statements: (hash) => rowStatements(site, draft, hash), + diagnose: check, + }; +} diff --git a/packages/core-ts/src/site-delete.ts b/packages/core-ts/src/site-delete.ts index 39a92dc3..3e0cce08 100644 --- a/packages/core-ts/src/site-delete.ts +++ b/packages/core-ts/src/site-delete.ts @@ -45,6 +45,7 @@ export const SITE_TABLES = [ "analytics_settings", "media_uploads", "asset_refs", + "entry_refs", "snapshot_blobs", "media_blobs", "approvals", diff --git a/packages/core-ts/src/store.ts b/packages/core-ts/src/store.ts index a0a273b3..1f5b8b35 100644 --- a/packages/core-ts/src/store.ts +++ b/packages/core-ts/src/store.ts @@ -59,6 +59,7 @@ import { type Cond, mapUniqueViolation, type Row } from "./node-sql.ts"; import type { Creatable, NodeKind, SiteNode } from "./nodes.ts"; import { BodyProposals } from "./proposals.ts"; import type { RedirectMatch } from "./redirects.ts"; +import type { FieldTypeSupport } from "./references/definitions.ts"; import { type PublishReleaseInput, type PublishReleaseResult, @@ -148,6 +149,7 @@ export class SiteGraphStore { this.blocks = options.blocks ?? createBlockRegistry(); this.themeSettings = options.themeSettings; this.themeDesign = options.themeDesign; + this.fieldTypes = options.fieldTypes; this.#onPublished = options.onPublished; this.#onHookError = options.onHookError ?? ((e) => console.error("onPublished failed:", e)); this.proposals = new BodyProposals({ @@ -212,13 +214,11 @@ export class SiteGraphStore { readonly blocks: BlockRegistry; /** RFC-0017: the theme's design settings, when it declares any. */ readonly themeSettings: PropsSchema | undefined; - /** RFC-0021: the theme's DESIGN.md, when it ships one. */ - readonly themeDesign: string | undefined; + readonly themeDesign: string | undefined; // RFC-0021: the theme's DESIGN.md, when it ships one + /** RFC-0045 §1: reads reference and repeater fields (absent: both types refused). */ + readonly fieldTypes: FieldTypeSupport | undefined; - /** - * Whether `availability` may be written for this site (S15): the renderer serving it has - * confirmed it enforces gates (`version` 2: derived gates, RFC-0042 §3.6). Imports ask it too. - */ + /** S15: the renderer confirmed it enforces gates (`version` 2: derived gates). Imports ask too. */ async gatesEnforced(siteId: string, version = 1): Promise { return (await this.#gates?.enforced(siteId, version)) === true; } diff --git a/packages/core-ts/src/store/entries.ts b/packages/core-ts/src/store/entries.ts index fc360d70..7d7f2b0c 100644 --- a/packages/core-ts/src/store/entries.ts +++ b/packages/core-ts/src/store/entries.ts @@ -11,12 +11,16 @@ import { fillMissingSrc } from "../media/refs.ts"; import { and, casUpdateStatement, insertNodeStatement, OpBatch, runOpBatch } from "../node-sql.ts"; import { uniquenessKey } from "../nodes.ts"; import { entryPath } from "../permalinks.ts"; +import { planLeave } from "../references/leave.ts"; +import { dropEntryRefsStatement } from "../references/rebuild.ts"; +import { canonicalReferences } from "../references/writer.ts"; import { canonicalTerms } from "../taxonomy.ts"; import { entersCollection, planNewCollection } from "./collection-paths.ts"; import type { StoreContext } from "./context.ts"; import { normalizeEntryFields, planEntryFields } from "./entry-fields.ts"; +import { planEntryIntegrity } from "./entry-integrity.ts"; import { planSourceForward } from "./source-forward.ts"; -import { checkLocalizedNode, checkTranslationLinks } from "./translations.ts"; +import { checkLocalizedNode } from "./translations.ts"; import type { NodeWriteOptions } from "./types.ts"; import { checkBodyDirectives } from "./validation.ts"; @@ -60,11 +64,14 @@ export class EntryEngine { draft = corrections ? { ...rest, corrections } : (rest as ContentEntry); } draft = normalizeEntryFields(site, draft); + // RFC-0045 §1: a member's id is stored as its group's. + draft = await canonicalReferences(this.#ctx.sql, site, { draft, stored }); assertValid("content-entry", canonicalTerms(draft)); checkLocalizedNode("contentEntry", site, draft); - await checkTranslationLinks(this.#ctx, { kind: "contentEntry", site, node: draft, stored }); // RFC-0041 §3: typed field values, requiredToPublish and their asset references. const fields = await planEntryFields(this.#ctx, site, draft, stored); + // RFC-0033 links; RFC-0045 §2 targets, anchors, `entry_refs`, a move or group change. + const integrity = await planEntryIntegrity(this.#ctx, site, { draft, stored }); if (stored) await assertUnlocked(this.#ctx.sql, stored.siteId, [stored.id], options.releaseEdit); if (draft.body.format === "markdown") { @@ -101,13 +108,18 @@ export class EntryEngine { const opening = entersCollection(stored, entry) ? await planNewCollection(this.#ctx, site, entry.collection, entry.id) : undefined; - const guard = and(prepared.guard, refs.guard, opening?.guard, fields.guard); + const guard = and(prepared.guard, refs.guard, opening?.guard, fields.guard, integrity.guard); const hash = await nodeHash(entry); // RFC-0041 §5: a change to non-localized values alone leaves translations fresh. const forward = await planSourceForward(this.#ctx, site, { stored, entry, ts }); // RFC-0042 §4: list values before source-forward, which changes the translations' hashes. const lists = await planListValues(this.#ctx.sql, site, { stored, entry, at: ts }); - const own = [...refs.statements(hash), ...fields.statements(hash), ...lists(hash)]; + const own = [ + ...refs.statements(hash), + ...fields.statements(hash), + ...integrity.statements(hash), + ...lists(hash), + ]; const extra = [...own, ...(await forward(hash))]; const key = uniquenessKey("contentEntry", entry); const sameLocale = stored?.locale === entry.locale; @@ -133,6 +145,7 @@ export class EntryEngine { await this.#ctx.tax.diagnoseEntry(entry); await refs.diagnose(); await fields.diagnose(); + await integrity.diagnose(entry); await opening?.diagnose(); throw new ConflictError("contentEntry", entry.id, stored?.rev ?? 0, rev); } @@ -152,26 +165,36 @@ export class EntryEngine { await this.#ctx.tax.diagnoseEntry(entry); await refs.diagnose(); await fields.diagnose(); + await integrity.diagnose(entry); await opening?.diagnose(); }, }); return entry; } - /** CAS delete with its asset references and list values (RFC-0042 §4), in one batch. */ + /** + * CAS delete with its asset references, list values (RFC-0042 §4) and reference rows, in one + * batch; refused while it would strand a reference a published entry shows (RFC-0045 §2.2). + */ async delete(entry: ContentEntry): Promise { await assertUnlocked(this.#ctx.sql, entry.siteId, [entry.id]); const site = await this.#ctx.graph.getSite(entry.siteId); const lists = await planListValuesDelete(this.#ctx.sql, site, entry, this.#ctx.stamp()); + const leave = site ? await planLeave(this.#ctx.sql, site, { stored: entry }) : undefined; + const guard = leave ? ` AND ${leave.guard.sql}` : ""; const [r] = await this.#ctx.sql.batch([ { - sql: "DELETE FROM nodes WHERE id = ? AND rev = ? AND kind = 'contentEntry' AND site_id = ?", - params: [entry.id, entry.rev, entry.siteId], + sql: `DELETE FROM nodes WHERE id = ? AND rev = ? AND kind = 'contentEntry' AND site_id = ?${guard}`, + params: [entry.id, entry.rev, entry.siteId, ...(leave?.guard.params ?? [])], }, this.#ctx.assets.dropRefsStatement(entry.id), + dropEntryRefsStatement(entry.id), ...lists, ]); - if (r?.changes !== 1) - throw new ConflictError("contentEntry", entry.id, entry.rev, await this.#ctx.revOf(entry.id)); + if (r?.changes !== 1) { + const rev = await this.#ctx.revOf(entry.id); + if (rev === entry.rev) await leave?.diagnose(); + throw new ConflictError("contentEntry", entry.id, entry.rev, rev); + } } } diff --git a/packages/core-ts/src/store/entry-fields.ts b/packages/core-ts/src/store/entry-fields.ts index 3a025a48..02763deb 100644 --- a/packages/core-ts/src/store/entry-fields.ts +++ b/packages/core-ts/src/store/entry-fields.ts @@ -58,6 +58,8 @@ async function originalOf( const t = entry.translation; if (!t || t.group === entry.id || t.source?.siteId !== undefined) return undefined; const original = await ctx.graph.getContentEntry(t.group); + // RFC-0045 OQ 14: an original that was relinked into another group is nobody's original. + if ((original?.translation?.group ?? original?.id) !== t.group) return undefined; return original?.siteId === entry.siteId ? original : undefined; } diff --git a/packages/core-ts/src/store/entry-integrity.ts b/packages/core-ts/src/store/entry-integrity.ts new file mode 100644 index 00000000..0c227758 --- /dev/null +++ b/packages/core-ts/src/store/entry-integrity.ts @@ -0,0 +1,46 @@ +/** + * RFC-0033 and RFC-0045 §2 for an entry write, planned together: the translation links and their + * group guards (`checkTranslationLinks`), the reference rules (`planEntryRefs`: targets, the + * definitions anchor, `entry_refs`), and the delete-like rules of a move or group change + * (`planLeave`). One guard, one set of anchored statements, one diagnosis. + */ +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import { and, type Cond } from "../node-sql.ts"; +import { planLeave } from "../references/leave.ts"; +import { planEntryRefs } from "../references/writer.ts"; +import type { StoreContext } from "./context.ts"; +import { checkTranslationLinks } from "./translations.ts"; + +export interface EntryIntegrityPlan { + readonly guard: Cond | undefined; + /** `entry_refs` rows, anchored on the entry's new hash. */ + statements(hash: string): SqlStatement[]; + /** After a write that matched no row: the specific refusal, if one applies now. */ + diagnose(entry: ContentEntry): Promise; +} + +export async function planEntryIntegrity( + ctx: StoreContext, + site: Site, + write: { draft: ContentEntry; stored: ContentEntry | undefined }, +): Promise { + const { draft, stored } = write; + const linked = await checkTranslationLinks(ctx, { + kind: "contentEntry", + site, + node: draft, + stored, + }); + const references = await planEntryRefs(ctx.sql, site, draft, stored); + const leave = stored ? await planLeave(ctx.sql, site, { stored, next: draft }) : undefined; + return { + guard: and(references.guard, linked, leave?.guard), + statements: (hash) => references.statements(hash), + diagnose: async (entry) => { + await references.diagnose(); + await leave?.diagnose(); + await checkTranslationLinks(ctx, { kind: "contentEntry", site, node: entry, stored }); + }, + }; +} diff --git a/packages/core-ts/src/store/pages.ts b/packages/core-ts/src/store/pages.ts index 769d5f1b..e770e6b8 100644 --- a/packages/core-ts/src/store/pages.ts +++ b/packages/core-ts/src/store/pages.ts @@ -45,7 +45,7 @@ export class PageEngine { checkLocalizedNode("page", site, draft); // RFC-0042 §4: field queries run against the site this batch's site CAS writes over. checkPageQueries(site, draft); - await checkTranslationLinks(this.#ctx, { kind: "page", site, node: draft }); + const linked = await checkTranslationLinks(this.#ctx, { kind: "page", site, node: draft }); await this.#ctx.checkGates(site.id, undefined, draft.availability); const refs = await this.#ctx.assets.planRefs("page", site.id, draft, undefined); const page = refs.node; @@ -67,7 +67,7 @@ export class PageEngine { }); const batch = new OpBatch(this.#ctx.newId(ID_PREFIX.operation), site.id, ts); await batch.update("site", site, nextSite); - await batch.insert("page", page, and(pageGuard(this.#ctx, page), refs.guard)); + await batch.insert("page", page, and(pageGuard(this.#ctx, page), refs.guard, linked)); batch.append(...refs.statements(await nodeHash(page))); await plan?.emit(batch); await runOpBatch(this.#ctx.sql, batch, { @@ -104,7 +104,12 @@ export class PageEngine { const siteGuard = checkPageQueries(site, next, stored) ? hashIs(site.id, await nodeHash(site)) : undefined; - await checkTranslationLinks(this.#ctx, { kind: "page", site, node: next, stored }); + const linked = await checkTranslationLinks(this.#ctx, { + kind: "page", + site, + node: next, + stored, + }); const sameLocale = stored.locale === next.locale; const plan = await this.#ctx.tax.planPathChange(site, ts, { locale: next.locale, @@ -122,7 +127,7 @@ export class PageEngine { "page", next, page.rev, - and(pageGuard(this.#ctx, next), refs.guard, siteGuard), + and(pageGuard(this.#ctx, next), refs.guard, siteGuard, linked), ), "page", key, @@ -143,7 +148,7 @@ export class PageEngine { "page", stored, next, - and(pageGuard(this.#ctx, next), refs.guard, siteGuard), + and(pageGuard(this.#ctx, next), refs.guard, siteGuard, linked), ); batch.append(...extra); await plan.emit(batch); diff --git a/packages/core-ts/src/store/site-fields.ts b/packages/core-ts/src/store/site-fields.ts index 4093a32c..b2293f41 100644 --- a/packages/core-ts/src/store/site-fields.ts +++ b/packages/core-ts/src/store/site-fields.ts @@ -16,6 +16,11 @@ import { } from "../fields/index.ts"; import { sameJson } from "../fields/types.ts"; import { and, type Cond } from "../node-sql.ts"; +import { referenceDefinitionOptions } from "../references/capability.ts"; +import { + type ReferenceDataChange, + referenceDefinitionChanges, +} from "../references/definition-changes.ts"; import type { StoreContext } from "./context.ts"; /** What a site write's definition changes add: the guard that keeps "no entry uses it" true. */ @@ -84,23 +89,46 @@ export async function planFieldDefinitions( if (!stored?.collections && !next.collections) return undefined; // Only a change is checked: other site writes never re-run the caps on stored settings. if (stored && sameJson(stored.collections, next.collections)) return undefined; - refuseFieldIssues(checkFieldDefinitions(next, stored)); + const options = await referenceDefinitionOptions(ctx.graph, next, stored !== undefined); + refuseFieldIssues(checkFieldDefinitions(next, stored, options)); if (!stored) return undefined; + // RFC-0045 §2.8: reference and repeater changes that depend on stored values. + const references = referenceDefinitionChanges(next.id, stored, next); + refuseFieldIssues(references.refused); + const guards = await checkReferenceUsage(ctx, references.data); // A collection that goes away entirely is RFC-0040's (`collection_in_use` while it has entries). const changes = fieldDefinitionChanges(stored, next).filter( (c): c is DataChange => (c.change === "removed" || c.change === "optionsRemoved") && next.collections?.[c.collection] !== undefined, ); - if (changes.length === 0) return undefined; + if (changes.length === 0 && !guards) return undefined; await checkUsage(ctx, next.id, changes); return { guard: and( + guards, ...changes.map((c) => { const q = usage(next.id, c); return { sql: `(${q.sql}) = 0`, params: q.params }; }), ), - diagnose: () => checkUsage(ctx, next.id, changes), + diagnose: async () => { + await checkUsage(ctx, next.id, changes); + await checkReferenceUsage(ctx, references.data); + }, }; } + +/** Counts each reference change's stranded values; throws when any; returns the batch guard. */ +async function checkReferenceUsage( + ctx: StoreContext, + changes: readonly ReferenceDataChange[], +): Promise { + const issues: FieldIssue[] = []; + for (const c of changes) { + const n = Number((await ctx.sql.get<{ n: number }>(c.count.sql, c.count.params))?.n ?? 0); + if (n > 0) issues.push(c.issue(n)); + } + refuseFieldIssues(issues); + return and(...changes.map((c) => ({ sql: `(${c.count.sql}) = 0`, params: c.count.params }))); +} diff --git a/packages/core-ts/src/store/translations.ts b/packages/core-ts/src/store/translations.ts index bc5f233b..4ba573f0 100644 --- a/packages/core-ts/src/store/translations.ts +++ b/packages/core-ts/src/store/translations.ts @@ -7,6 +7,7 @@ import type { ContentEntry, Page, Site, TranslationMethod } from "@cloudhal/core-contracts"; import type { SqlDriver } from "@cloudhal/storage"; import { InvariantError } from "../errors.ts"; +import { and, type Cond } from "../node-sql.ts"; import type { SiteGraphStore } from "../store.ts"; type Localized = Page | ContentEntry; @@ -97,20 +98,79 @@ export function checkLocalizedNode(kind: LocalizedKind, site: Site, node: Locali ); } -/** Invariants 2–3 that read the graph: the source exists, belongs to the group, stays in the project; one node per locale. */ +/** + * RFC-0045 §1, §2.2: the guards a write that sets or changes a node's translation group carries. + * + * - **Written group.** A write whose `translation.group` (or `source`) is set or changed joins + * group X: X's node, if it is in this site, still has its own id as its group (R15). Omitted when + * X is the node's own id (a translation becoming an original), where it would read the + * pre-update row and fail every retry. + * - **Dependants.** A node whose id other same-site nodes use as their group cannot leave it + * (`translation_group_in_use`): its translations move first, so a group never loses its original + * while members still inherit from it. + */ +function groupGuards( + kind: LocalizedKind, + siteId: string, + write: { node: Localized; stored?: Localized | undefined; joins: boolean }, +): { written?: Cond; dependants?: Cond } { + const { node, stored } = write; + const group = node.translation?.group ?? node.id; + const out: { written?: Cond; dependants?: Cond } = {}; + if (write.joins && group !== node.id) + out.written = { + sql: `NOT EXISTS (SELECT 1 FROM nodes WHERE id = ? AND site_id = ? + AND json_extract(body, '$.translation.group') IS NOT NULL AND json_extract(body, '$.translation.group') <> ?)`, + params: [group, siteId, group], + }; + if (stored && (stored.translation?.group ?? stored.id) === stored.id && group !== stored.id) + out.dependants = { + sql: `NOT EXISTS (SELECT 1 FROM nodes WHERE site_id = ? AND kind = ? + AND json_extract(body, '$.translation.group') = ? AND id <> ?)`, + params: [siteId, kind, stored.id, stored.id], + }; + return out; +} + +async function holds(sql: SqlDriver, cond: Cond | undefined): Promise { + if (!cond) return true; + return (await sql.get<{ ok: number }>(`SELECT ${cond.sql} AS ok`, cond.params))?.ok === 1; +} + +/** + * Invariants 2–3 that read the graph: the source exists, belongs to the group, stays in the + * project; one node per locale. Returns the batch guard of RFC-0045's group rules + * (`groupGuards`), checked here first; undefined when the write changes no group. + */ export async function checkTranslationLinks( { graph, sql }: { graph: SiteGraphStore; sql: SqlDriver }, write: { kind: LocalizedKind; site: Site; node: Localized; stored?: Localized | undefined }, -): Promise { +): Promise { const { kind, site, node, stored } = write; const t = node.translation; - if (!t) return; - const source = t.source; const before = stored?.translation; + const source = t?.source; const sourceChanged = source !== undefined && (before?.source?.id !== source.id || before.source.siteId !== source.siteId); - if (source && sourceChanged) { + // RFC-0045 §2.2: a group-only change is checked like a source change. + const groupChanged = t !== undefined && before?.group !== t.group; + const guards = groupGuards(kind, site.id, { + node, + stored, + joins: sourceChanged || groupChanged, + }); + if (!(await holds(sql, guards.dependants))) + throw new InvariantError( + "translation_group_in_use", + `other language versions use ${node.id} as their group; move or unlink them first`, + { id: node.id }, + ); + if (!(await holds(sql, guards.written))) + fail("translation.group names a node that is itself a translation", { group: t?.group }); + const guard = and(guards.written, guards.dependants); + if (!t) return guard; + if (source && (sourceChanged || groupChanged)) { const sourceSiteId = source.siteId ?? site.id; if (sourceSiteId !== site.id) { const other = await graph.getSite(sourceSiteId); @@ -131,7 +191,7 @@ export async function checkTranslationLinks( if ((src.translation?.group ?? src.id) !== t.group) fail("translation.group is the source's group (its original's id)", { group: t.group }); } - if (before?.group === t.group && stored?.locale === node.locale) return; + if (before?.group === t.group && stored?.locale === node.locale) return guard; const taken = await sql.get<{ id: string }>( `SELECT id FROM nodes WHERE site_id = ? AND kind = ? AND id != ? AND json_extract(body, '$.translation.group') = ? AND json_extract(body, '$.locale') = ?`, @@ -143,6 +203,7 @@ export async function checkTranslationLinks( `the group already has a ${node.locale} version (${taken.id}); update that one instead`, { id: taken.id, locale: node.locale }, ); + return guard; } /** Every language version of `group` across the sites of `site`'s project, with staleness. */ diff --git a/packages/core-ts/src/store/types.ts b/packages/core-ts/src/store/types.ts index a0fadb5d..5fb4ff50 100644 --- a/packages/core-ts/src/store/types.ts +++ b/packages/core-ts/src/store/types.ts @@ -4,6 +4,7 @@ import type { BlobStore, SqlDriver } from "@cloudhal/storage"; import type { GateSupport } from "../availability.ts"; import type { BlockRegistry, PropsSchema } from "../blocks.ts"; import type { BlockWriter } from "../body-index.ts"; +import type { FieldTypeSupport } from "../references/definitions.ts"; import type { SearchProvider } from "../search-provider.ts"; export interface SiteGraphStoreOptions { @@ -50,6 +51,12 @@ export interface SiteGraphStoreOptions { readonly themeSettings?: PropsSchema; /** RFC-0021 §1: the theme's DESIGN.md (`src/DESIGN.md`), read by agents with the site's rules. */ readonly themeDesign?: string; + /** + * RFC-0045 §1: whether the renderer serving a site reads `reference` and `repeater` fields. + * Without it both types are refused on every write path (`fields_unsupported`); the shared + * suites pass one, production stores pass none until rollout step 7. + */ + readonly fieldTypes?: FieldTypeSupport; } /** diff --git a/packages/core-ts/src/sync-bundle.ts b/packages/core-ts/src/sync-bundle.ts index 11c48bdc..b7f3b547 100644 --- a/packages/core-ts/src/sync-bundle.ts +++ b/packages/core-ts/src/sync-bundle.ts @@ -6,6 +6,7 @@ */ import type { BlobStore, SqlDriver } from "@cloudhal/storage"; import type { GateSupport } from "./availability.ts"; +import type { FieldTypeSupport } from "./references/definitions.ts"; export interface StoragePair { sql: SqlDriver; @@ -33,6 +34,11 @@ export interface CopySiteOptions { * published takes the bundle's, as a first deploy does. */ published?: "source" | "target"; + /** + * RFC-0045 §1: whether the target's renderer reads reference and repeater fields. Without it a + * bundle defining them is refused (`fields_unsupported`), unlike `gates`. + */ + fieldTypes?: FieldTypeSupport; } export interface CopySiteResult { diff --git a/packages/core-ts/src/sync-plan.ts b/packages/core-ts/src/sync-plan.ts index 6196dd1a..7f139b43 100644 --- a/packages/core-ts/src/sync-plan.ts +++ b/packages/core-ts/src/sync-plan.ts @@ -17,6 +17,8 @@ import { canonicalJson, sha256Hex } from "./hash.ts"; import { checkHosts, hostStatements, hostsOf, type SiteHost } from "./hosts.ts"; import { importListStatements } from "./list-values/site-plan.ts"; import { refsRebuild } from "./media/refs-rebuild.ts"; +import { type FieldTypeSupport, referenceTypesSupported } from "./references/definitions.ts"; +import { entryRefsRebuild } from "./references/rebuild.ts"; import { type BundleMedia, type NodeRow, @@ -41,6 +43,7 @@ export interface ImportPlanOptions { mode?: "bootstrap" | "replace" | undefined; gates?: GateSupport | undefined; published?: "source" | "target" | undefined; + fieldTypes?: FieldTypeSupport | undefined; /** * Return a bootstrap conflict (`conflict`) and missing media (`media.missing`) instead of * throwing them (`cloudhal import` before it copies media, and `--dry-run`). Every other @@ -183,6 +186,35 @@ async function bodiesToWrite( return out; } +/** + * The site bodies the bundle's *new* snapshots name that the target already holds. Node bodies + * are content-addressed and shared by every site, so a site body held for a site whose renderer + * reads the RFC-0045 types could be re-pointed by another site's new snapshot: the gate checks + * them too. Snapshots the target has were accepted already, and a capable site needs no read. + * Unreadable ones are left to `assertSnapshotsBacked`. + */ +async function heldSiteBodies( + to: StoragePair, + siteHashes: ReadonlySet, + gate: { written: readonly VerifiedBody[]; support: FieldTypeSupport | undefined; siteId: string }, +): Promise<{ hash: string; body: unknown }[]> { + const { written, support, siteId } = gate; + if (siteHashes.size === 0 || (await referenceTypesSupported(siteId, support))) return []; + const mine = new Set(written.map((b) => b.hash)); + const out: { hash: string; body: unknown }[] = []; + for (const hash of siteHashes) { + if (mine.has(hash) || !HASH.test(hash)) continue; + const blob = await to.blobs.get(`nodes/${hash}.json`); + if (!blob) continue; + try { + out.push({ hash, body: JSON.parse(new TextDecoder().decode(blob)) }); + } catch { + // Not JSON: not a site body this gate can read; the snapshot's own checks stand. + } + } + return out; +} + /** Every snapshot must be fully backed by blobs: the bundle's or the target's. */ async function assertSnapshotsBacked( to: StoragePair, @@ -297,9 +329,12 @@ function nodeBatch( batch.push({ sql: "DELETE FROM nodes WHERE site_id = ?", params: [siteId] }); deleted = changes.target.length; for (const n of changes.rows) { + // RFC-0045 §2.1: bodies are stored canonically, as store writes are, so the definitions + // and collections anchors compare equal text (the hash is over canonical JSON already). + const body = canonicalJson(JSON.parse(n.body)); batch.push({ sql: "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - params: [n.id, n.site_id, n.kind, n.key, n.rev, n.hash, n.body, n.created_at, n.updated_at], + params: [n.id, n.site_id, n.kind, n.key, n.rev, n.hash, body, n.created_at, n.updated_at], }); inserted++; } @@ -308,6 +343,8 @@ function nodeBatch( // only then: beside an unchanged node set its plan-time rows could replace a concurrent // write's fresher ones (#196). Its chunked `INSERT`s count against the cap below. batch.push(...refsRebuild(siteId, changes.rows)); + // RFC-0045 §2.4: `entry_refs` likewise, under exactly the same condition (R10). + batch.push(...entryRefsRebuild(siteId, changes.rows)); } const written = changes.rows.find((n) => n.kind === "site" && n.id === siteId) ?? siteRow; batch.push(...hostStatements(extra.hosts, { siteId, hash: written.hash })); @@ -340,7 +377,7 @@ export async function prepareImport( ): Promise<{ plan: ImportPlan; writes: ImportWrites }> { const mode = options.mode ?? "bootstrap"; const { siteId } = bundle; - const siteRow = await checkBundle(bundle, options.gates); + const siteRow = await checkBundle(bundle, options.gates, options.fieldTypes); await assertTimeZone(to, siteId, bundle); const down = await takenDown(to, siteId); const media = await mediaOnTarget(to, bundle, { report: options.report === true, down }); @@ -350,14 +387,24 @@ export async function prepareImport( await assertIdsFree(to, bundle); const refs = bundle.snapshots.map((s) => ({ id: s.id, refs: snapshotRefs(s) })); const blobs = await bodiesToWrite(to, bundle, refs); - // RFC-0045 §1: the site bodies of snapshots too, once verified (rollout step 2 passes the - // capability, CopySiteOptions.fieldTypes, here and to checkBundle). + // RFC-0045 §1: the site bodies of snapshots too, once verified, and the held ones they name. const siteHashes = new Set( refs.flatMap((s) => s.refs.filter((r) => r.kind === "site").map((r) => r.hash)), ); - await assertSnapshotFieldTypes(siteId, blobs, siteHashes, undefined); - await assertSnapshotsBacked(to, bundle, refs); const snapshots = await snapshotsToAdd(to, bundle); + const added = new Set(snapshots.ids); + const newSiteHashes = new Set( + refs + .filter((s) => added.has(s.id)) + .flatMap((s) => s.refs.filter((r) => r.kind === "site").map((r) => r.hash)), + ); + const held = await heldSiteBodies(to, newSiteHashes, { + written: blobs, + support: options.fieldTypes, + siteId, + }); + await assertSnapshotFieldTypes(siteId, [...blobs, ...held], siteHashes, options.fieldTypes); + await assertSnapshotsBacked(to, bundle, refs); const project = await projectToAdd(to, bundle); const changes = await nodeChanges(to, bundle, mode, options); // RFC-0042 §4: the list values start over with the node set (rebuilt after the batch). diff --git a/packages/core-ts/src/testing-reference-gate.ts b/packages/core-ts/src/testing-reference-gate.ts index 76370d47..97f1a449 100644 --- a/packages/core-ts/src/testing-reference-gate.ts +++ b/packages/core-ts/src/testing-reference-gate.ts @@ -126,7 +126,7 @@ export function referenceGateSuite(ctx: SuiteContext): void { expect((await store().getSite(site.id))?.collections).toEqual(site.collections); }); - it("refuses a field operation that turns a field into one", async () => { + it("refuses a field operation that turns a field into one (step 4 adds them)", async () => { const site = await setup(); for (const type of ["reference", "repeater"] as const) expect( @@ -138,7 +138,7 @@ export function referenceGateSuite(ctx: SuiteContext): void { { actor: person }, ), ), - ).toBe("fields_unsupported"); + ).toBe("field_operation_unsupported"); expect((await store().getSite(site.id))?.collections).toEqual(site.collections); }); diff --git a/packages/core-ts/src/testing-references.ts b/packages/core-ts/src/testing-references.ts new file mode 100644 index 00000000..63c82da2 --- /dev/null +++ b/packages/core-ts/src/testing-references.ts @@ -0,0 +1,851 @@ +/** + * RFC-0045 rollout step 2: the store and integrity of reference fields, on every backend + * (`describeSiteGraphStore`). The types stay refused in production; this suite enables them with + * `SiteGraphStoreOptions.fieldTypes` on a store of its own. + * + * Covered: `entry_refs` rows and canonical group ids; the target guard (missing, cross-site, + * wrong-collection, own-group, archived targets); the definitions and collections anchors; the + * delete, move and group-change rules with the "shown by a published entry" policy; the relink + * refusal and the written-group guard; `field_operation_running`; rebuilds in imports; and the + * races R1 to R16 of §2.3 as interleaved batches (`racingDriver`). R8, R9 and R14 are noted where + * they stand: R8 and R14 are accepted, R9 needs the operations of rollout step 4. + */ +import type { Actor, ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { describe, expect, it } from "vitest"; +import { GATES_ENFORCED } from "./availability.ts"; +import { ConflictError, InvariantError } from "./errors.ts"; +import { runFieldOperation } from "./fields-operations/index.ts"; +import { canonicalJson, nodeHash } from "./hash.ts"; +import { SiteGraphStore } from "./store.ts"; +import { exportSite, importSite } from "./sync.ts"; +import { racingDriver, type SuiteContext } from "./testing-taxonomy.ts"; + +const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; +const L = (label: string) => ({ en: { label } }); +const ENABLED = { supported: () => true }; + +const VENUE: FieldDefinition = { + key: "venue", + type: "reference", + labels: L("Venue"), + localized: false, + collections: ["places", "halls"], +}; +const SPONSOR: FieldDefinition = { + key: "sponsor", + type: "reference", + labels: L("Sponsor"), + localized: true, + collections: ["people"], +}; +const SPEAKERS: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["people"] }, + { key: "talk", type: "text", labels: L("Talk") }, + ], +}; +const ROOM: FieldDefinition = { + key: "room", + type: "reference", + labels: L("Room"), + localized: false, + collections: ["places"], +}; + +const COLLECTIONS: NonNullable = { + events: { fields: [VENUE, SPONSOR, SPEAKERS] }, + talks: { fields: [ROOM] }, + places: {}, + halls: {}, + people: {}, +}; + +async function code(p: Promise): Promise { + try { + await p; + } catch (e) { + if (e instanceof InvariantError) return e.code; + if (e instanceof ConflictError) return "conflict"; + throw e; + } + return "ok"; +} + +interface RefRow { + target_id: string; + field: string; + root_key: string; + shared: number; + source_status: string; +} + +export function referencesSuite(ctx: SuiteContext): void { + describe("RFC-0045 step 2: entry_refs and reference integrity", () => { + let n = 0; + let tick = 0; + const now = () => new Date(Date.UTC(2026, 10, 11, 9, 0, tick++)); + const make = (sql = ctx.sql()) => + new SiteGraphStore({ + sql, + blobs: ctx.blobs(), + now, + gates: GATES_ENFORCED, + fieldTypes: ENABLED, + }); + const store = () => make(); + /** A store whose first matching write first runs `hook` (the batch that commits first). */ + const racing = (hook: () => Promise, when?: (q: string) => boolean) => + make(racingDriver(ctx.sql(), hook, when)); + const storage = () => ({ sql: ctx.sql(), blobs: ctx.blobs() }); + + async function world(collections: Site["collections"] | null = COLLECTIONS): Promise { + n++; + const project = await store().createProject({ + name: "Refs", + slug: `refs-${n}`, + ownerId: person.id, + siteIds: [], + }); + return store().createSite({ + projectId: project.id, + name: "Refs", + slug: `refs-site-${n}`, + defaultLocale: "en", + locales: ["en", "tr"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + ...(collections ? { collections } : {}), + }); + } + + const entry = (site: Site, collection: string, over: Partial = {}) => + store().createContentEntry({ + siteId: site.id, + collection, + slug: `${collection}-${Math.random().toString(36).slice(2, 8)}`, + locale: "en", + status: "draft", + title: `A ${collection}`, + body: { format: "markdown", value: "Body" }, + fields: {}, + tags: [], + author: person, + ...over, + } as never); + + const sourceOf = async (e: ContentEntry) => ({ + id: e.id, + locale: e.locale, + rev: e.rev, + hash: await nodeHash(e), + }); + const translate = async (e: ContentEntry, over: Partial = {}) => + entry((await store().getSite(e.siteId)) as Site, e.collection, { + locale: "tr", + slug: `${e.slug}-tr`, + translation: { group: e.id, method: "human", source: await sourceOf(e) }, + ...over, + }); + + const rows = (sourceId: string) => + ctx + .sql() + .all>( + "SELECT target_id, field, root_key, shared, source_status FROM entry_refs WHERE source_id = ? ORDER BY field, target_id", + [sourceId], + ); + const fresh = async (e: ContentEntry) => (await store().getContentEntry(e.id)) as ContentEntry; + const update = (e: ContentEntry, over: Partial) => + store().updateContentEntry({ ...e, ...over }); + /** Writes a site body directly: a field operation's phase A, as its own batch would. */ + async function rawSite(siteId: string, change: (s: Site) => Site) { + const site = (await store().getSite(siteId)) as Site; + const next = { ...change(site), rev: site.rev + 1 }; + await ctx + .sql() + .run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + site.id, + ]); + } + const withFields = (site: Site, collection: string, fields: FieldDefinition[]): Site => ({ + ...site, + collections: { + ...site.collections, + [collection]: { ...site.collections?.[collection], fields } as never, + }, + }); + + describe("writes", () => { + it("writes one row per stored value, deduplicated after canonical group ids", async () => { + const site = await world(); + const hall = await entry(site, "places", { status: "published" }); + const hallTr = await translate(hall, { status: "published" }); + const ada = await entry(site, "people"); + const event = await entry(site, "events", { + status: "published", + fields: { + // the Turkish member's id is stored as its group's + venue: hallTr.id, + speakers: [ + { person: ada.id, talk: "Opening" }, + { person: ada.id, talk: "Closing" }, + ], + }, + }); + expect(event.fields?.venue).toBe(hall.id); + expect(await rows(event.id)).toEqual([ + { + target_id: ada.id, + field: "speakers.person", + root_key: "speakers", + shared: 1, + source_status: "published", + }, + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "published", + }, + ]); + // every write rewrites the rows: the status follows, a removed value goes + const draft = await update(event, { status: "draft", fields: { venue: hall.id } }); + expect(await rows(draft.id)).toEqual([ + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "draft", + }, + ]); + await store().deleteContentEntry(await fresh(draft)); + expect(await rows(draft.id)).toEqual([]); + }); + + it("refuses missing, cross-site, wrong-collection, own-group and archived new targets", async () => { + const site = await world(); + const other = await world(); + const foreign = await entry(other, "places"); + const person1 = await entry(site, "people"); + const archived = await entry(site, "places", { status: "archived" }); + const event = await entry(site, "events"); + const set = (fields: Record) => update(event, { fields }); + expect(await code(set({ venue: "entry_01HZX8K3QMISSING" }))).toBe("unknown_entry"); + // another site's id is refused exactly like one that does not exist + expect(await code(set({ venue: foreign.id }))).toBe("unknown_entry"); + expect(await code(set({ venue: person1.id }))).toBe("reference_target_moved"); + expect(await code(set({ venue: archived.id }))).toBe("unknown_entry"); + expect(await code(set({ speakers: [{ person: event.id }] }))).toBe("invalid_fields"); + expect(await rows(event.id)).toEqual([]); + }); + + it("keeps an existing value whose target was archived, and refuses publishing a dangling one", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + await update(hall, { status: "archived" }); + // unchanged values are not re-checked by a draft write; going live checks every own value + const titled = await update(event, { title: "Renamed" }); + expect(await code(update(titled, { status: "published" }))).toBe("ok"); + const live = await fresh(event); + await update(live, { status: "draft" }); + await store().deleteContentEntry(await fresh(hall)); + expect(await code(update(await fresh(event), { status: "published" }))).toBe( + "unknown_entry", + ); + // the row is kept for the dangling value (the invariant), and blocks once a member returns + expect((await rows(event.id)).map((r) => r.target_id)).toEqual([hall.id]); + }); + }); + + describe("delete, move and group change", () => { + it("a published referrer blocks deleting the last usable member; a draft one does not", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const draft = await entry(site, "events", { fields: { venue: hall.id } }); + const live = await entry(site, "events", { + status: "published", + fields: { venue: hall.id }, + }); + const e = await code(store().deleteContentEntry(await fresh(hall))); + expect(e).toBe("entry_referenced"); + try { + await store().deleteContentEntry(await fresh(hall)); + } catch (err) { + const details = (err as InvariantError).details as { + referrers: { id: string }[]; + total: number; + }; + expect(details.referrers.map((r) => r.id)).toEqual([live.id]); + expect(details.total).toBe(1); + } + // archiving is allowed, and does not release the lock (decision 2) + await update(await fresh(hall), { status: "archived" }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + await update(live, { status: "draft" }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + expect(await rows(draft.id)).toHaveLength(1); + }); + + it("another usable member in an allowed collection lets the delete go", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const hallTr = await translate(hall); + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + // a translation is a member: deleting the original keeps the group in the site + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + expect(await code(store().deleteContentEntry(await fresh(hallTr)))).toBe( + "entry_referenced", + ); + }); + + it("a member in a collection the field does not allow does not count", async () => { + const site = await world(); + const hall = await entry(site, "places"); + await translate(hall, { collection: "people" } as never); + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + }); + + it("moves: refused out of every allowed collection, allowed into another one", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const talk = await entry(site, "talks", { status: "published", fields: { room: hall.id } }); + expect(await code(update(hall, { collection: "halls" }))).toBe("reference_target_moved"); + await update(talk, { status: "draft" }); + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + // venue allows halls: the move keeps it satisfied by the node itself + expect(await code(update(await fresh(hall), { collection: "halls" }))).toBe("ok"); + expect(await code(update(await fresh(hall), { collection: "people" }))).toBe( + "reference_target_moved", + ); + }); + + it("group changes: leaving a referenced group is refused; a relinked node leaves it", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const other = await entry(site, "places", { locale: "tr" }); + await entry(site, "events", { status: "published", fields: { venue: other.id } }); + // relinking `other` as a translation of `hall` leaves its own group + const relink = { + translation: { group: hall.id, method: "human" as const, source: await sourceOf(hall) }, + }; + expect(await code(update(other, relink))).toBe("entry_referenced"); + }); + + it("relink with a draft referrer, then publish the referrer (refused), then delete", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const other = await entry(site, "places", { locale: "tr" }); + const event = await entry(site, "events", { fields: { venue: other.id } }); + const relinked = await update(other, { + translation: { group: hall.id, method: "human", source: await sourceOf(hall) }, + }); + expect(relinked.translation?.group).toBe(hall.id); + // the old group id has no member any more: by id alone the node is not a member + expect(await code(update(event, { status: "published" }))).toBe("unknown_entry"); + expect(await code(store().deleteContentEntry(await fresh(relinked)))).toBe("ok"); + }); + + it("refuses relinking an original that has translations (translation_group_in_use)", async () => { + const site = await world(); + const hall = await entry(site, "places"); + await translate(hall); + const other = await entry(site, "places", { locale: "tr", slug: "other" }); + const relink = { + translation: { group: other.id, method: "human" as const, source: await sourceOf(other) }, + }; + expect(await code(update(hall, { ...relink, locale: "en" }))).toBe( + "translation_group_in_use", + ); + }); + + it("a translation becoming its own original is not refused by the written-group guard", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const tr = await translate(hall); + const own = { translation: { group: tr.id, method: "original" as const } }; + expect(await code(update(tr, own))).toBe("ok"); + }); + + it("a delete or move on a site without collections binds SQL NULL in the anchor", async () => { + const site = await world(null); + const post = await entry(site, "posts"); + expect(await code(update(post, { collection: "notes" }))).toBe("ok"); + expect(await code(store().deleteContentEntry(await fresh(post)))).toBe("ok"); + }); + }); + + describe("imports", () => { + it("stores canonical bodies, so the anchors compare equal text", async () => { + const site = await world(); + const bundle = await exportSite(storage(), site.id); + const copy = structuredClone(bundle); + const row = copy.nodes.find((r) => r.kind === "site"); + if (!row) throw new Error("no site row"); + // the same document with its keys in another order: same hash, other text + const parsed = JSON.parse(row.body) as Record; + row.body = JSON.stringify(Object.fromEntries(Object.entries(parsed).reverse())); + row.rev += 0; + const moved = copy.nodes.find((r) => r.kind !== "site"); + if (moved) moved.updated_at = "2026-11-11T00:00:00.000Z"; + await importSite(storage(), copy, { mode: "replace", fieldTypes: ENABLED }); + const stored = await ctx + .sql() + .get<{ body: string }>("SELECT body FROM nodes WHERE id = ?", [site.id]); + expect(stored?.body).toBe(canonicalJson(parsed)); + const hall = await entry(site, "places"); + expect(await code(entry(site, "events", { fields: { venue: hall.id } }))).toBe("ok"); + }); + + it("a new snapshot of a site without the capability cannot point at a held site body that defines them", async () => { + const capable = await world(); + const plain = await world({ notes: {} }); + const only = { supported: (id: string) => id === capable.id }; + const held = await store().createSnapshot(capable.id, { actor: person, reason: "manual" }); + const heldSite = held.nodes.find((r) => r.kind === "site"); + const own = await store().createSnapshot(plain.id, { actor: person, reason: "manual" }); + const bundle = await exportSite(storage(), plain.id); + const row = bundle.snapshots.find((r) => r.id === own.id); + if (!row || !heldSite) throw new Error("no snapshot"); + const forged = { ...own, id: "snap_01HZX8K3QFORGE" }; + forged.nodes = own.nodes.map((r) => + r.kind === "site" ? { ...r, hash: heldSite.hash } : r, + ); + bundle.snapshots.push({ ...row, id: forged.id, body: canonicalJson(forged) }); + expect( + await code(importSite(storage(), bundle, { mode: "replace", fieldTypes: only })), + ).toBe("fields_unsupported"); + expect(await store().getSnapshot(forged.id)).toBeUndefined(); + }); + }); + + describe("field operations (§2.6)", () => { + it("refuses operations that name a reference or repeater field (step 4)", async () => { + const site = await world(); + for (const op of [ + { op: "rename_field", collection: "events", from: "venue", to: "place" }, + { op: "remove_field", collection: "events", key: "speakers" }, + ] as const) + expect(await code(runFieldOperation(store(), site.id, op, { actor: person }))).toBe( + "field_operation_unsupported", + ); + }); + }); + + describe("races (§2.3): the batch that commits second fails, then retries or refuses", () => { + it("R1: a referrer write and deleting the target's last member", async () => { + const site = await world(); + const hall = await entry(site, "places"); + // the referrer commits first: the delete's batch-time condition sees its row + const del = racing(async () => { + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + }); + expect(await code(del.deleteContentEntry(hall))).toBe("entry_referenced"); + // the delete commits first: the referrer's target guard sees no member + const hall2 = await entry(site, "places"); + const event = await entry(site, "events"); + const write = racing(async () => { + await store().deleteContentEntry(await fresh(hall2)); + }); + expect( + await code(write.updateContentEntry({ ...event, fields: { venue: hall2.id } })), + ).toBe("unknown_entry"); + expect(await rows(event.id)).toEqual([]); + }); + + it("R2: a referrer write and moving the target to a disallowed collection", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const talk = await entry(site, "talks"); + const write = racing(async () => { + await update(await fresh(hall), { collection: "people" }); + }); + expect(await code(write.updateContentEntry({ ...talk, fields: { room: hall.id } }))).toBe( + "reference_target_moved", + ); + const hall2 = await entry(site, "places"); + const move = racing(async () => { + await entry(site, "talks", { status: "published", fields: { room: hall2.id } }); + }); + expect(await code(move.updateContentEntry({ ...hall2, collection: "people" }))).toBe( + "reference_target_moved", + ); + }); + + it("R3: a referrer write and adding a reference field over its key (definitions anchor)", async () => { + const site = await world({ notes: {}, places: {} }); + const hall = await entry(site, "places"); + const note = await entry(site, "notes"); + // the site write commits first: the entry write planned without definitions conflicts + const write = racing(async () => { + const s = (await store().getSite(site.id)) as Site; + await store().updateSite(withFields(s, "notes", [ROOM])); + }); + expect(await code(write.updateContentEntry({ ...note, fields: { room: hall.id } }))).toBe( + "conflict", + ); + // on retry the definition is known, and the value gets its row + await update(await fresh(note), { fields: { room: hall.id } }); + expect((await rows(note.id)).map((r) => r.field)).toEqual(["room"]); + // adding a field over a key entries hold is refused + const plain = await world({ notes: {}, places: {} }); + await entry(plain, "notes", { fields: { room: "entry_01HZX8K3QTEXT00" } }); + const s = (await store().getSite(plain.id)) as Site; + expect(await code(store().updateSite(withFields(s, "notes", [ROOM])))).toBe( + "field_key_in_use", + ); + }); + + it("R3: removing an allowed collection while a value's group has a member there", async () => { + const site = await world(); + const hall = await entry(site, "halls"); + await entry(site, "events", { fields: { venue: hall.id } }); + const narrowed = withFields(site, "events", [ + { ...VENUE, collections: ["places"] } as never, + SPONSOR, + SPEAKERS, + ]); + expect(await code(store().updateSite(narrowed))).toBe("field_change_needs_operation"); + expect( + await code( + store().updateSite( + withFields(site, "events", [ + { ...VENUE, multiple: true } as never, + SPONSOR, + SPEAKERS, + ]), + ), + ), + ).toBe("field_change_needs_operation"); + }); + + it("R4, R5: a move or delete and a concurrent definitions change (collections anchor)", async () => { + const site = await world(); + const hall = await entry(site, "places"); + let extra = 0; + const edit = async () => { + const s = (await store().getSite(site.id)) as Site; + extra++; + await store().updateSite({ + ...s, + collections: { ...s.collections, [`extra${extra}`]: {} }, + }); + }; + expect(await code(racing(edit).updateContentEntry({ ...hall, collection: "halls" }))).toBe( + "conflict", + ); + expect(await code(racing(edit).deleteContentEntry(await fresh(hall)))).toBe("conflict"); + // a publish changes only settings outside `collections`: no conflict + const publish = async () => { + const s = (await store().getSite(site.id)) as Site; + const snap = await store().createSnapshot(s.id, { actor: person, reason: "manual" }); + await store().publish(s, snap.id); + }; + expect(await code(racing(publish).deleteContentEntry(await fresh(hall)))).toBe("ok"); + }); + + it("R6: changing a member's group and a referrer write", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const other = await entry(site, "places", { locale: "tr" }); + const event = await entry(site, "events", { status: "published" }); + const relink = racing(async () => { + await update(await fresh(event), { fields: { venue: other.id } }); + }); + const translation = { + group: hall.id, + method: "human" as const, + source: await sourceOf(hall), + }; + expect(await code(relink.updateContentEntry({ ...other, translation }))).toBe( + "entry_referenced", + ); + // the other order: the relinked node no longer matches its old group by id + const other2 = await entry(site, "places", { locale: "tr", slug: "other-2" }); + const hall2 = await entry(site, "places", { slug: "hall-2" }); + const write = racing(async () => { + await update(await fresh(other2), { + translation: { group: hall2.id, method: "human", source: await sourceOf(hall2) }, + }); + }); + expect( + await code( + write.updateContentEntry({ ...(await fresh(event)), fields: { venue: other2.id } }), + ), + ).toBe("unknown_entry"); + }); + + it("R7: a referrer becoming published and deleting its target", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const del = racing(async () => { + await update(await fresh(event), { status: "published" }); + }); + expect(await code(del.deleteContentEntry(hall))).toBe("entry_referenced"); + const hall2 = await entry(site, "places"); + const event2 = await entry(site, "events", { fields: { venue: hall2.id } }); + const publish = racing(async () => { + await store().deleteContentEntry(await fresh(hall2)); + }); + expect(await code(publish.updateContentEntry({ ...event2, status: "published" }))).toBe( + "unknown_entry", + ); + }); + + it("R8 (accepted): a translation inheriting a deleted target goes live and shows nothing", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const tr = await translate(event); + await store().deleteContentEntry(await fresh(hall)); + expect(await code(update(tr, { status: "published" }))).toBe("ok"); + }); + + it("the shared rule: who a value blocks through", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const ada = await entry(site, "people"); + // a draft original's localized: false value blocks once a published translation inherits it + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const tr = await translate(event); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + const hall2 = await entry(site, "places"); + await update(await fresh(event), { fields: { venue: hall2.id } }); + await update(await fresh(tr), { status: "published" }); + expect(await code(store().deleteContentEntry(await fresh(hall2)))).toBe("entry_referenced"); + // a localized: true value: a draft translation's own value never blocks, and a published + // translation with its own value does not inherit the original's + const own = await entry(site, "people"); + await update(await fresh(event), { fields: { venue: hall2.id, sponsor: ada.id } }); + const t2 = await fresh(tr); + await update(t2, { fields: { sponsor: own.id } }); + expect(await code(store().deleteContentEntry(await fresh(ada)))).toBe("ok"); + expect(await code(store().deleteContentEntry(await fresh(own)))).toBe("entry_referenced"); + expect((await rows(tr.id)).map((r) => r.shared)).toEqual([2]); + }); + + it("R10: an unchanged import does not rebuild; a written node set does", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const bundle = await exportSite(storage(), site.id); + // a referrer write between the unchanged import's plan and its batch keeps its rows + const racingStorage = { + sql: racingDriver( + ctx.sql(), + async () => { + await update(await fresh(event), { status: "published" }); + }, + () => true, + ), + blobs: ctx.blobs(), + }; + const result = await importSite(racingStorage, bundle, { fieldTypes: ENABLED }); + expect(result.nodes.unchanged).toBe(true); + expect((await rows(event.id))[0]?.source_status).toBe("published"); + // the bundle differs now, so `replace` writes the node set and rebuilds from its bodies + await importSite(storage(), bundle, { mode: "replace", fieldTypes: ENABLED }); + expect(await rows(event.id)).toEqual([ + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "draft", + }, + ]); + // without the capability the bundle is refused (the gate of step 1) + expect(await code(importSite(storage(), bundle))).toBe("fields_unsupported"); + }); + + it("R11: rows are kept for a missing target and block once a member reappears", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const tr = await translate(hall); + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + await store().deleteContentEntry(await fresh(hall)); + // the translation is the group's last member here + expect(await code(store().deleteContentEntry(await fresh(tr)))).toBe("entry_referenced"); + }); + + it("R12: deletes wait while a text → reference conversion runs (field_operation_running)", async () => { + const site = await world({ + ...COLLECTIONS, + notes: { fields: [{ key: "where", type: "text", labels: L("Where"), localized: false }] }, + }); + const hall = await entry(site, "places"); + const marker = (s: Site) => + withFields(s, "notes", [ + { + ...ROOM, + key: "where", + migration: { id: "snap_01HZX8K3Q9MG", fromType: "text" }, + } as never, + ]); + // a delete planned before phase A commits fails on the collections anchor… + const del = racing(() => rawSite(site.id, marker)); + expect(await code(del.deleteContentEntry(hall))).toBe("conflict"); + // …and its retry is refused until the marker is gone + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe( + "field_operation_running", + ); + expect(await code(update(await fresh(hall), { collection: "halls" }))).toBe( + "field_operation_running", + ); + }); + + it("R13: a blocking row from a field the plan did not see forces a retry", async () => { + const site = await world(); + const place = await entry(site, "places"); + await translate(place, { collection: "halls" } as never); + await entry(site, "events", { status: "published", fields: { venue: place.id } }); + // venue allows halls, so the hall member keeps it; talks.room allows places only + const del = racing(async () => { + await entry(site, "talks", { status: "published", fields: { room: place.id } }); + }); + expect(await code(del.deleteContentEntry(place))).toBe("entry_referenced"); + }); + + it("R15: creating a translation of X and relinking X", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const other = await entry(site, "places", { locale: "tr", slug: "other" }); + const target = { group: other.id, method: "human" as const, source: await sourceOf(other) }; + // the translation commits first: the relink sees a dependant + const relink = racing(async () => { + await translate(await fresh(hall), { locale: "tr", slug: "hall-tr" }); + }); + expect(await code(relink.updateContentEntry({ ...hall, translation: target }))).toBe( + "translation_group_in_use", + ); + // the relink commits first: the joining write's written-group guard fails + const hall2 = await entry(site, "places", { slug: "hall-2" }); + const source = await sourceOf(hall2); + const join = racing(async () => { + const other2 = await entry(site, "places", { locale: "tr", slug: "other-2" }); + await update(await fresh(hall2), { + translation: { group: other2.id, method: "human", source: await sourceOf(other2) }, + }); + }); + const created = join.createContentEntry({ + siteId: site.id, + collection: "places", + slug: "hall-2-de", + locale: "tr", + status: "draft", + title: "Salon", + body: { format: "markdown", value: "Body" }, + fields: {}, + tags: [], + author: person, + translation: { group: hall2.id, method: "human", source }, + } as never); + expect(await code(created)).toBe("invalid_translation"); + }); + + it("R16: a title-only write racing rename_field phase A keeps its rows, through fromKey", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const marker = { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }; + const renamed = (s: Site) => + withFields(s, "events", [ + { ...VENUE, key: "place", migration: marker } as never, + SPONSOR, + SPEAKERS, + ]); + await racing(() => rawSite(site.id, renamed)).updateContentEntry({ + ...event, + title: "Renamed", + }); + expect(await rows(event.id)).toEqual([ + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "draft", + }, + ]); + }); + + it("R16: a new field reusing a running rename's old key: the exact key wins in shared", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const both = (s: Site) => + withFields(s, "events", [ + { + ...VENUE, + key: "place", + migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, + } as never, + { ...VENUE, localized: true }, + SPONSOR, + SPEAKERS, + ]); + await racing(() => rawSite(site.id, both)).updateContentEntry({ ...event, title: "Both" }); + expect((await rows(event.id))[0]?.shared).toBe(2); + }); + + it("R16: a title-only write racing set_field_localized phase A reads the new shared", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const flipped = (s: Site) => + withFields(s, "events", [ + { + ...VENUE, + localized: true, + migration: { id: "snap_01HZX8K3Q9MG", fromLocalized: false }, + } as never, + SPONSOR, + SPEAKERS, + ]); + await racing(() => rawSite(site.id, flipped)).updateContentEntry({ + ...event, + title: "Flip", + }); + expect((await rows(event.id))[0]?.shared).toBe(2); + }); + + it("R16: a key with no definition at batch time fails the batch, never drops the row", async () => { + const site = await world(); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const dropped = (s: Site) => withFields(s, "events", [SPONSOR, SPEAKERS]); + const lost = racing(() => rawSite(site.id, dropped)); + expect(await code(lost.updateContentEntry({ ...event, title: "Lost" }))).toBe("conflict"); + expect(await rows(event.id)).toHaveLength(1); + expect((await fresh(event)).title).toBe(event.title); + }); + + it("a foreign archive does not pin another tenant's entries", async () => { + const site = await world(); + const victim = await world(); + const hall = await entry(victim, "places"); + const event = await entry(site, "events", { status: "published" }); + // a row naming another site's id (as a hand-made archive could leave) pins nothing there + await ctx + .sql() + .run( + "INSERT INTO entry_refs (site_id, target_id, source_id, source_collection, field, root_key, shared, source_status) VALUES (?, ?, ?, 'events', 'venue', 'venue', 1, 'published')", + [site.id, hall.id, event.id], + ); + expect(await code(store().deleteContentEntry(hall))).toBe("ok"); + }); + }); + }); +} diff --git a/packages/core-ts/src/testing.ts b/packages/core-ts/src/testing.ts index 4fdc30b8..2c3d9d00 100644 --- a/packages/core-ts/src/testing.ts +++ b/packages/core-ts/src/testing.ts @@ -18,6 +18,7 @@ import { listingsSuite } from "./testing-listings.ts"; import { menuSuite } from "./testing-menu.ts"; import { permalinksSuite } from "./testing-permalinks.ts"; import { referenceGateSuite } from "./testing-reference-gate.ts"; +import { referencesSuite } from "./testing-references.ts"; import { racingDriver, type SuiteContext, taxonomySuite } from "./testing-taxonomy.ts"; export interface StoreFactory { @@ -277,6 +278,7 @@ export function describeSiteGraphStore(label: string, factory: StoreFactory): vo fieldsSuite(suite); fieldOperationsSuite(suite); referenceGateSuite(suite); + referencesSuite(suite); listingsSuite(suite); entryQuerySuite(suite); }); diff --git a/packages/core-ts/test/references-store.test.ts b/packages/core-ts/test/references-store.test.ts new file mode 100644 index 00000000..26ddc3ee --- /dev/null +++ b/packages/core-ts/test/references-store.test.ts @@ -0,0 +1,187 @@ +/** + * RFC-0045 rollout step 2: the parts of the reference store that need SQL but no store — the + * anchors' text equality over canonical bodies, the rows a body yields, the chunked rebuild and + * the `entry_refs.shared` NOT NULL mapping. better-sqlite3 only; the shared suite + * (`testing-references.ts`) runs the behaviour on every backend. + */ +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { migrate, migrations } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { describe, expect, it } from "vitest"; +import { ConflictError } from "../src/errors.ts"; +import { canonicalJson } from "../src/hash.ts"; +import { REFS_STATEMENT_BYTES } from "../src/media/refs-rebuild.ts"; +import { mapUniqueViolation } from "../src/node-sql.ts"; +import { canonicalizeSiteRows } from "../src/references/canonical-rows.ts"; +import { entryRefsRebuild } from "../src/references/rebuild.ts"; +import { entryRefRows } from "../src/references/rows.ts"; +import { collectionsAnchor, definitionsAnchor } from "../src/references/sql.ts"; + +const FIXTURES = join(import.meta.dirname, "../../core-contracts/fixtures/valid/site"); +const L = (label: string) => ({ en: { label } }); + +async function db() { + const sql = await BetterSqlite3Driver.open(":memory:"); + await migrate(sql, migrations); + return sql; +} + +async function holds(site: Site, stored: unknown = site) { + const sql = await db(); + await sql.run( + "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, 'site', ?, 0, 'h', ?, 't', 't')", + [site.id, site.id, site.id, canonicalJson(stored)], + ); + const check = async (c: { sql: string; params: readonly unknown[] }) => + (await sql.get<{ ok: number }>(`SELECT ${c.sql} AS ok`, c.params as never))?.ok; + return { sql, check }; +} + +describe("the anchors compare canonical bodies as text", () => { + const sites = readdirSync(FIXTURES) + .filter((f) => f.endsWith(".json")) + .map((f) => [f, JSON.parse(readFileSync(join(FIXTURES, f), "utf8")) as Site] as const); + + it.each(sites)( + "%s: every collection's definitions and the collections round-trip", + async (_, site) => { + const { check } = await holds(site); + expect(await check(collectionsAnchor(site))).toBe(1); + for (const key of Object.keys(site.collections ?? {})) + expect(await check(definitionsAnchor(site, key)), key).toBe(1); + // a collection without definitions, and an absent `collections`, bind SQL NULL + expect(await check(definitionsAnchor(site, "nowhere"))).toBe(1); + }, + ); + + it("fractional and large bounds survive the stored form", async () => { + const fields: FieldDefinition[] = [ + { + key: "ratio", + type: "number", + labels: L("Ratio"), + localized: false, + minimum: 0.1, + maximum: 1e21, + }, + { key: "tiny", type: "number", labels: L("Tiny"), localized: false, minimum: -2.5e-7 }, + ]; + const site = { + id: "site_01HZX8K3Q9AB", + collections: { things: { fields } }, + } as unknown as Site; + const { check } = await holds(site); + expect(await check(definitionsAnchor(site, "things"))).toBe(1); + expect(await check(collectionsAnchor(site))).toBe(1); + }); + + it("an absent collections binds NULL, a changed one fails", async () => { + const site = { id: "site_01HZX8K3Q9AC" } as Site; + const { check } = await holds(site); + expect(await check(collectionsAnchor(site))).toBe(1); + expect(await check(collectionsAnchor({ ...site, collections: {} }))).toBe(0); + const other = await holds({ ...site, collections: { a: {} } } as Site); + expect(await other.check(collectionsAnchor(site))).toBe(0); + }); +}); + +describe("rows of a body", () => { + const venue: FieldDefinition = { + key: "venue", + type: "reference", + labels: L("Venue"), + localized: true, + multiple: true, + collections: ["places"], + }; + const speakers: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["people"] }, + { key: "talk", type: "text", labels: L("Talk") }, + ], + }; + + it("one row per (field, target), keys read through a running rename", () => { + const renamed = { + ...venue, + key: "place", + migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, + }; + const rows = entryRefRows([renamed, speakers], { + venue: ["entry_01HZX8K3QAAA", "entry_01HZX8K3QBBB"], + speakers: [ + { person: "entry_01HZX8K3QAAA" }, + { person: "entry_01HZX8K3QAAA", talk: "x" }, + "junk", + ], + }); + expect(rows).toEqual([ + { t: "entry_01HZX8K3QAAA", f: "venue", k: "venue", shared: 2 }, + { t: "entry_01HZX8K3QBBB", f: "venue", k: "venue", shared: 2 }, + { t: "entry_01HZX8K3QAAA", f: "speakers.person", k: "speakers", shared: 1 }, + ]); + }); + + it("the rebuild's INSERTs stay below 90 KB", () => { + const site = { + id: "site_01HZX8K3Q9AD", + kind: "site", + body: canonicalJson({ + id: "site_01HZX8K3Q9AD", + collections: { events: { fields: [venue] } }, + }), + }; + const entries = Array.from({ length: 3000 }, (_, i) => ({ + id: `entry_${String(i).padStart(12, "0")}`, + kind: "contentEntry", + body: canonicalJson({ + collection: "events", + status: "published", + fields: { venue: [`entry_T${String(i).padStart(11, "0")}`] }, + }), + })); + const statements = entryRefsRebuild(site.id, [site, ...entries]); + expect(statements[0]?.sql).toMatch(/^DELETE FROM entry_refs WHERE site_id = \?/); + expect(statements.length).toBeGreaterThan(2); + const encoder = new TextEncoder(); + for (const s of statements) { + const bytes = encoder.encode(s.sql + (s.params ?? []).join("")).length; + expect(bytes).toBeLessThan(REFS_STATEMENT_BYTES); + } + }); +}); + +describe("the shared NOT NULL mapping", () => { + it("maps only entry_refs.shared to a ConflictError", () => { + const shared = new Error("NOT NULL constraint failed: entry_refs.shared"); + expect(mapUniqueViolation(shared, "contentEntry", "k")).toBeInstanceOf(ConflictError); + const other = new Error("NOT NULL constraint failed: entry_refs.source_status"); + expect(mapUniqueViolation(other, "contentEntry", "k")).toBe(other); + const d1 = new Error( + "D1_ERROR: NOT NULL constraint failed: entry_refs.shared: SQLITE_CONSTRAINT", + ); + expect(mapUniqueViolation(d1, "contentEntry", "k")).toBeInstanceOf(ConflictError); + }); +}); + +describe("canonical site rows (§2.4)", () => { + it("rewrites a verbatim site row once, under its hash, and leaves canonical ones alone", async () => { + const sql = await db(); + const site = { id: "site_01HZX8K3Q9CN", rev: 3, collections: { b: {}, a: { fields: [] } } }; + const verbatim = JSON.stringify(site); + await sql.run( + "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, 'site', ?, 3, 'h', ?, 't', 't')", + [site.id, site.id, site.id, verbatim], + ); + expect(await canonicalizeSiteRows(sql)).toEqual({ read: 1, rewritten: 1 }); + const row = await sql.get<{ body: string; hash: string }>("SELECT body, hash FROM nodes"); + expect(row).toEqual({ body: canonicalJson(site), hash: "h" }); + expect(await canonicalizeSiteRows(sql)).toEqual({ read: 1, rewritten: 0 }); + }); +}); diff --git a/packages/mcp-server/src/field-hints.ts b/packages/mcp-server/src/field-hints.ts index fab8f971..02e8a103 100644 --- a/packages/mcp-server/src/field-hints.ts +++ b/packages/mcp-server/src/field-hints.ts @@ -44,6 +44,20 @@ const HINTS: Partial> = { "a reference field's collections must be collections of this site (list_collections)", too_many_references: "an entry holds at most 100 references across its fields and repeater items; remove some", + unknown_entry: + "each reference names the groupId of an entry of this site in one of the field's collections (list_entries); remove or replace the ids at the paths in details.issues", + reference_target_moved: + "published entries show this entry through a field that does not allow the new collection: keep it where it is, or ask a person to unpublish or edit those entries first", + entry_referenced: + "published entries (details.referrers) show this entry, and it is the last one of its group here: ask a person to unpublish or edit them first; archiving it does not release them", + translation_group_in_use: + "other language versions use this entry as their original: unlink or move them first, then change its translation", + field_operation_running: + "a field operation is converting values into references to this collection: wait until it finishes, then try again", + field_operation_unsupported: + "field operations on reference and repeater fields are not available on this server yet; edit the entries instead", + field_key_in_use: + "entries of the collection already hold a value under that key: pick another key, or convert the existing field with change_field_type", list_field_in_use: "contentList sections on the pages in details.pages still filter or sort by that field: change their where/orderBy first (update_page), then try again", }; diff --git a/packages/storage/migrations/0033_entry_refs.ts b/packages/storage/migrations/0033_entry_refs.ts new file mode 100644 index 00000000..a84e5a57 --- /dev/null +++ b/packages/storage/migrations/0033_entry_refs.ts @@ -0,0 +1,35 @@ +import type { Migration } from "../src/migrate.ts"; + +/** + * RFC-0045 §2: `entry_refs`, the reverse index of reference values (the `asset_refs` pattern, + * RFC-0004 §4.4). One row per (holding node, field path, stored translation group id), whatever + * the target's state; core-ts writes them in each entry's own batch, anchored on the entry's new + * hash (AGENTS.md rule 4), and rebuilds them beside an imported node set. Every guard and lookup + * filters `site_id`, so a row naming another tenant's id pins nothing there. + * + * - `target_id`: the referenced translation group id, as stored; + * - `source_id`, `source_collection`: the node holding the value and its collection; + * - `field`: `venue`, or `speakers.person` inside a repeater; `root_key` the top-level key; + * - `shared`: from the definition, 1 for `localized: false` (every inheriting member shows it), + * 2 for `localized: true` (only members without their own value); + * - `source_status`: the holding node's own status. + * + * DDL only: no reference value can exist before rollout step 7, on any path (§1). + */ +export const migration0033EntryRefs: Migration = { + name: "0033_entry_refs", + sql: ` +CREATE TABLE entry_refs ( + site_id TEXT NOT NULL, + target_id TEXT NOT NULL, + source_id TEXT NOT NULL, + source_collection TEXT NOT NULL, + field TEXT NOT NULL, + root_key TEXT NOT NULL, + shared INTEGER NOT NULL, + source_status TEXT NOT NULL, + PRIMARY KEY (source_id, field, target_id) +); +CREATE INDEX entry_refs_target ON entry_refs (site_id, target_id, source_status); +`, +}; diff --git a/packages/storage/migrations/index.ts b/packages/storage/migrations/index.ts index 40860008..4ce3127b 100644 --- a/packages/storage/migrations/index.ts +++ b/packages/storage/migrations/index.ts @@ -31,6 +31,7 @@ import { migration0029EntryListValues } from "./0029_entry_list_values.ts"; import { migration0030RateLimits } from "./0030_rate_limits.ts"; import { migration0031Oauth } from "./0031_oauth.ts"; import { migration0032OauthClientKindIndex } from "./0032_oauth_client_kind_index.ts"; +import { migration0033EntryRefs } from "./0033_entry_refs.ts"; /** All Cloudhal storage migrations, in order. */ export const migrations: readonly Migration[] = [ @@ -66,4 +67,5 @@ export const migrations: readonly Migration[] = [ migration0030RateLimits, migration0031Oauth, migration0032OauthClientKindIndex, + migration0033EntryRefs, ]; diff --git a/packages/storage/test/migrate.test.ts b/packages/storage/test/migrate.test.ts index dc6191cf..8afc66b1 100644 --- a/packages/storage/test/migrate.test.ts +++ b/packages/storage/test/migrate.test.ts @@ -50,6 +50,7 @@ describe("migrate", () => { "0030_rate_limits", "0031_oauth", "0032_oauth_client_kind_index", + "0033_entry_refs", ]); expect((await migrate(driver, migrations)).applied).toEqual([]); expect(await appliedMigrations(driver)).toEqual([ @@ -85,6 +86,7 @@ describe("migrate", () => { "0030_rate_limits", "0031_oauth", "0032_oauth_client_kind_index", + "0033_entry_refs", ]); const tables = await driver.all<{ name: string }>( "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", @@ -121,6 +123,7 @@ describe("migrate", () => { "comments", "entry_list_state", "entry_list_values", + "entry_refs", "invitation", "jwks", "media_blobs", @@ -209,6 +212,7 @@ describe("migrate", () => { "0030_rate_limits", "0031_oauth", "0032_oauth_client_kind_index", + "0033_entry_refs", ]); expect(await appliedMigrations(driver)).toEqual([ "0001_init", @@ -243,6 +247,7 @@ describe("migrate", () => { "0030_rate_limits", "0031_oauth", "0032_oauth_client_kind_index", + "0033_entry_refs", ]); await h.dispose(); }); @@ -284,6 +289,7 @@ describe("migrate", () => { "0030_rate_limits", "0031_oauth", "0032_oauth_client_kind_index", + "0033_entry_refs", ]); const ok = await driver.get("SELECT name FROM sqlite_master WHERE name = 'ok'"); expect(ok).toBeUndefined(); From d1078959193379f9a757bf4584378b9effdea5df Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:31:59 +0300 Subject: [PATCH 2/6] docs: ADR-0046 and the step-2a changeset (RFC-0045, #114) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/reference-store-integrity.md | 23 +++++++++++++++++++ docs/adr/0046-references-and-repeaters.md | 21 +++++++++++++++++ docs/adr/README.md | 1 + ...ctions-stage-4-references-and-repeaters.md | 2 +- 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .changeset/reference-store-integrity.md create mode 100644 docs/adr/0046-references-and-repeaters.md diff --git a/.changeset/reference-store-integrity.md b/.changeset/reference-store-integrity.md new file mode 100644 index 00000000..7503644a --- /dev/null +++ b/.changeset/reference-store-integrity.md @@ -0,0 +1,23 @@ +--- +"@cloudhal/storage": minor +"@cloudhal/core-ts": minor +"@cloudhal/api": patch +"@cloudhal/cloudflare": patch +"@cloudhal/mcp-server": patch +--- + +Reference fields get their index and integrity rules (RFC-0045 §2, rollout step 2a, #114). Both new field types stay refused on every write path in production, so no reference value can exist yet; the store and the shared suites run them through `SiteGraphStoreOptions.fieldTypes` and `CopySiteOptions.fieldTypes`. + +- **Storage.** Migration `0033_entry_refs` adds `entry_refs`, one row per stored reference value, removed with its site. +- **Store.** Entry writes keep the rows in their own batch and check their targets at batch time (`unknown_entry`, `reference_target_moved`). Deletes, moves and group changes are refused while a published entry shows a reference to the group's last usable member (`entry_referenced`, `reference_target_moved`, `field_operation_running`). Imports rebuild the rows beside a written node set. Field operations naming a reference or repeater field answer `field_operation_unsupported`. Definition changes that would strand values are refused (`field_key_in_use`, removed target collections or sub-fields, `multiple`). +- **API and MCP.** The new refusals answer 409 or 422, with MCP hints. +- **Cloudflare.** The hourly `nodes.canonical` job rewrites site rows that older imports stored verbatim, so the anchors compare equal text; local stores do the same when they open. + +**Behaviour changes on existing sites** (RFC-0045 decision 3): + +- relinking a node that other nodes use as their translation group is refused (`translation_group_in_use`, 409); +- `resolveFields` and the store read an original's non-localized values only when the original's own group is its id: a node relinked into another group lends them to nobody (OQ 14); +- a change of `translation.group` alone gets the same source-group check as a source change, and a write joining a group checks that its node is still that group's original; +- imports store node bodies in canonical form (the hash is unchanged); +- deletes, moves and group changes of entries carry the collections anchor, and entry writes that change `fields` or `collection` carry the definitions anchor: a concurrent change of the collections' settings makes them answer 409, and a retry succeeds; +- field operations that would turn a field into a reference or repeater answer `field_operation_unsupported` (409) instead of `fields_unsupported`. diff --git a/docs/adr/0046-references-and-repeaters.md b/docs/adr/0046-references-and-repeaters.md new file mode 100644 index 00000000..4e7e08a6 --- /dev/null +++ b/docs/adr/0046-references-and-repeaters.md @@ -0,0 +1,21 @@ +# ADR-0046: Collections stage 4: reference fields by translation group, guarded by an `entry_refs` index; repeaters of flat items +- Status: Accepted +- Date: 2026-11-11 (decided by Halil, RFC-0045 accepted 2026-11-10; lands with rollout step 2) +- Related: RFC-0045, RFC-0041 / ADR-0043, RFC-0042 / ADR-0044, RFC-0033 / ADR-0035, RFC-0004 (`asset_refs`), RFC-0008 / ADR-0023 (releases), #114, #71 +- Sources: [Sanity reference type](https://www.sanity.io/docs/studio/reference-type) · [Payload relationship field](https://payloadcms.com/docs/fields/relationship) · [Cloudflare D1 limits](https://developers.cloudflare.com/d1/platform/limits/) · [SQLite JSON functions](https://www.sqlite.org/json1.html) + +## Context +Collections have typed fields (ADR-0043), but a venue, a speaker or a brand is still plain text retyped on every entry, and a recipe's ingredients are lines of one text field. Two field types fix that, and each needs integrity rules that hold under D1, which has only atomic batches (AGENTS.md rule 4). + +## Decision +1. **Two types.** `reference` (one to five target collections of the same site, one target or up to 50) and `repeater` (up to 50 items of up to ten flat scalar sub-fields; no nesting). Both are refused on every write path (`fields_unsupported`) until rollout step 7, when the site Worker reports `fields: 2`. +2. **A reference stores a translation group id**, `COALESCE(translation.group, id)`, the one identity shared by every language version in every site. It resolves only through members of the page's own site, in the page's locale, then `localization.fallback`. Membership is that expression everywhere (guards, renderer, purges). +3. **`entry_refs`** holds one row per (holder, field path, stored group id), whatever the target's state, written in the holder's own batch, anchored on its new hash, with a plain `INSERT` (never `OR IGNORE`) that reads `shared` from the site row at batch time. Every guard and lookup filters `site_id`. +4. **Strong where the live site depends.** Only what a published entry shows locks: a delete, a move out of every allowed collection, or a group change of a group's last usable member is refused while a published entry shows a reference to it (`entry_referenced`, `reference_target_moved`). Archiving and unpublishing are always allowed; an unresolved reference renders nothing, and publish warns (`referenceIssues`) but never fails. +5. **Races are closed at batch time**, not by reads: the target guard in the referrer's CAS; the definitions anchor (a collection's definitions as planned) on entry writes that change values; the collections anchor on every delete, move and group change; planned pairs with an unseen-field check; the written-group and dependants guards on translation links (`translation_group_in_use`). The cases R1–R16 of RFC-0045 §2.3 are shared-suite tests. +6. **Rebuilds only beside a written node set** (imports, pulls, pushes, restores), chunked below 90 KB a statement. References are not release dependencies. + +## Consequences +- Existing sites see the phase-2 behaviour changes listed in RFC-0045's "Decisions" (relink refusal, the `readsOriginal` tightening, canonical import bodies, the anchors' 409s on concurrent settings changes). +- Purges read the published and previous snapshots' bodies (rollout step 3); no new cache tags. +- Per-sub-field localization, `datetime` and multiple values inside items, and sorting by a referenced entry's field stay out of stage 4 (RFC-0045 open questions 2 and 3, §4). diff --git a/docs/adr/README.md b/docs/adr/README.md index 17026bda..d48ae96e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -62,3 +62,4 @@ One file per decision, numbered, immutable once accepted (write a new ADR to sup | [0043](0043-typed-custom-fields.md) | Typed custom fields: nine closed types on `Site.collections..fields` (discriminator `type`), values in `ContentEntry.fields`, one eval-free checker (`@cloudhal/core-ts/fields`) for store, API and MCP that also emits the agents' JSON Schema, `requiredToPublish` at review and publish, per-field localization, definitions on the site node with caps, renames/type changes/removals only as snapshotted, approved operations (RFC-0041) | | [0044](0044-collections-stage-3.md) | Collections stage 3: JSON-LD presets (Event, JobPosting, Product, Recipe) mapped from fields through a closed table, expiry as a render-time gate (`unlist`/`hide`, no automation), listings filtered and sorted by field with one shared evaluator and no indexes yet, listing pages as ordinary pages and per-collection feeds through a middleware (RFC-0042) | | [0045](0045-mcp-authorization-server.md) | MCP authorization server: Better Auth `mcp()` + `jwt()` on the Studio database, issuer `/api/auth`, Ed25519 JWT access tokens (1 h, `aud` = `/mcp`) verified locally plus one `agent_grants` read per call, refresh rotation with a 30 s reuse window, consent on Studio's screen narrowed by the installation role, a token resolves to the same `Principal` as a key; no AS without a stable public origin (RFC-0044 Phase 2) | +| [0046](0046-references-and-repeaters.md) | Collections stage 4: `reference` fields store a translation group id resolved per site and locale, `repeater` fields hold up to 50 flat items; an `entry_refs` index written in each entry's batch guards deletes, moves and group changes only while a published entry shows the reference, with every race closed by batch-time anchors and guards (RFC-0045) | diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index 789780da..abcb4bba 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1,6 +1,6 @@ # RFC-0045: Collections stage 4: reference fields and repeaters -- Status: Accepted (2026-11-10, by Halil), with the three Owner decisions ratified; see "Decisions (2026-11-10)". The ADR lands with rollout step 2. +- Status: Accepted (2026-11-10, by Halil), with the three Owner decisions ratified; see "Decisions (2026-11-10)". ADR-0046 landed with rollout step 2. Drafted in PR #193 and revised six times after an independent review and five verification passes; the review log and the implementation watch list are at the end. - Author: Halil (Madraka), drafted with Claude · Date: 2026-11-10 - Related: From 008688a3b436bd0ad9df53c27f6a0eb8bc4d9ce2 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:46:05 +0300 Subject: [PATCH 3/6] fix(core-ts): read entry_refs.shared with an aggregate, not a correlated ORDER BY (PR #205 CI) workerd's SQLite (D1, Durable Objects) does not resolve the outer json_each alias p inside the shared subquery's ORDER BY (no such column: p.value). The exact key still wins over a running rename's fromKey, and no match still yields NULL, which fails the batch. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- packages/core-ts/src/references/writer.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/core-ts/src/references/writer.ts b/packages/core-ts/src/references/writer.ts index 5ee22b57..819f0b13 100644 --- a/packages/core-ts/src/references/writer.ts +++ b/packages/core-ts/src/references/writer.ts @@ -11,7 +11,8 @@ * collection's definitions as planned. * - **The rows**, anchored on the entry's new hash: an anchored `DELETE`, then one plain * `INSERT … SELECT` that reads `shared` from the site row as it is when the batch runs, by key - * or `migration.fromKey`. A key with no definition then yields `NULL` and fails the batch + * or `migration.fromKey` (the exact key wins; an aggregate, not a correlated `ORDER BY`, which + * workerd's SQLite does not resolve). A key with no definition then yields `NULL` and fails the batch * (`isSharedNullViolation` → `ConflictError`); it is never `OR IGNORE`. */ import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; @@ -228,12 +229,15 @@ function rowStatements(site: Site, draft: ContentEntry, hash: string): SqlStatem out.push({ sql: `INSERT INTO entry_refs (site_id, target_id, source_id, source_collection, field, root_key, shared, source_status) SELECT ?, json_extract(p.value, '$.t'), ?, ?, json_extract(p.value, '$.f'), json_extract(p.value, '$.k'), - (SELECT CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END + (SELECT COALESCE( + max(CASE WHEN json_extract(d.value, '$.key') = json_extract(p.value, '$.k') + THEN CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END END), + max(CASE WHEN json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k') + THEN CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END END)) FROM nodes s, json_each(json_extract(s.body, ?)) d WHERE s.id = ? AND s.kind = 'site' AND (json_extract(d.value, '$.key') = json_extract(p.value, '$.k') - OR json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k')) - ORDER BY json_extract(d.value, '$.key') = json_extract(p.value, '$.k') DESC LIMIT 1), + OR json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k'))), ? FROM json_each(?) p WHERE ${ANCHOR}`, params: [ From cb0ad735b7ed7d964af9029d97ec197722f90304 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:57:54 +0300 Subject: [PATCH 4/6] fix(core-ts): text anchors, anchored refs writes, fixed-size leave guard (PR #205 review) - A (#5, #3, #6, #7): the definitions and collections anchors read the fragment's stored text when planned (checked against the planned Site, else ConflictError) and compare that text at batch time, so a site row stored verbatim anchors like any other. The canonical-rows job and its local-open run are removed: nothing depends on canonical site rows. - B (#2, #4): in a site with reference fields every entry write carries the definitions anchor (status-only publishes included); the shared subquery only matches reference and repeater definitions, so a retyped key fails the batch. - C (#0): usable members are one json_each count over the needed collection lists (six parameters, per-pair semantics kept). - D (#8, #15): the list-values fingerprint format moves to 2, so rows built under the old readsOriginal rebuild once. - E (#16): a referrer write naming an entry outside the field's collections is unknown_entry (422); reference_target_moved stays the move's 409. - F (#12): a reference sub-field added over a key stored items hold is field_key_in_use, counted and guarded on the site CAS. - G (#1): racing cases for the R3 narrowing in both orders, R13 for a move and a group change, the #2 interleaving, a verbatim site row, 22 blocking pairs, and SQL tests of the shared subquery. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- packages/cloudflare/src/canonical-rows-job.ts | 28 -- packages/cloudflare/src/scheduler.ts | 2 - packages/core-ts/src/errors.ts | 4 +- packages/core-ts/src/fields/types.ts | 7 +- packages/core-ts/src/index.ts | 2 - .../core-ts/src/list-values/fingerprint.ts | 4 +- packages/core-ts/src/local.ts | 3 - .../core-ts/src/references/canonical-rows.ts | 46 --- .../src/references/definition-changes.ts | 32 +- packages/core-ts/src/references/leave.ts | 19 +- packages/core-ts/src/references/sql.ts | 97 ++++-- packages/core-ts/src/references/writer.ts | 41 ++- packages/core-ts/src/testing-references.ts | 276 ++++++++++++++---- .../core-ts/test/references-store.test.ts | 129 +++++--- 14 files changed, 454 insertions(+), 236 deletions(-) delete mode 100644 packages/cloudflare/src/canonical-rows-job.ts delete mode 100644 packages/core-ts/src/references/canonical-rows.ts diff --git a/packages/cloudflare/src/canonical-rows-job.ts b/packages/cloudflare/src/canonical-rows-job.ts deleted file mode 100644 index 1ab4e4f0..00000000 --- a/packages/cloudflare/src/canonical-rows-job.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * RFC-0045 §2.4: the admin Worker's `nodes.canonical` system job. Once an hour it rewrites site - * rows an older import stored verbatim in canonical form, one CAS each, so the definitions and - * collections anchors compare equal text (`canonicalizeSiteRows`). It reads one statement and - * writes at most `limit` more, which is its declared cost. - */ -import { canonicalizeSiteRows } from "@cloudhal/core-ts"; -import type { SystemJobDefinition } from "@cloudhal/schedules"; -import type { SqlDriver } from "@cloudhal/storage"; - -const HOUR_MS = 60 * 60_000; -const LIMIT = 10; - -/** The next full hour strictly after `after` (UTC). */ -const everyHour = (after: Date): Date => - new Date((Math.floor(after.getTime() / HOUR_MS) + 1) * HOUR_MS); - -export function canonicalRowsJob(sql: SqlDriver, log: (line: string) => void): SystemJobDefinition { - return { - name: "nodes.canonical", - cost: 1 + LIMIT, - next: everyHour, - async run() { - const r = await canonicalizeSiteRows(sql, { limit: LIMIT }); - if (r.rewritten > 0) log(`canonical rows: ${r.rewritten} site row(s) rewritten`); - }, - }; -} diff --git a/packages/cloudflare/src/scheduler.ts b/packages/cloudflare/src/scheduler.ts index a6b8a8d4..c1b40ab0 100644 --- a/packages/cloudflare/src/scheduler.ts +++ b/packages/cloudflare/src/scheduler.ts @@ -62,7 +62,6 @@ import { SqlScheduleStore } from "@cloudhal/schedules/sql"; import type { SqlDriver } from "@cloudhal/storage"; import { D1Driver } from "@cloudhal/storage/d1"; import { R2BlobStore } from "@cloudhal/storage/r2"; -import { canonicalRowsJob } from "./canonical-rows-job.ts"; import { flushCommentPurges, runCommentsHourly } from "./comments.ts"; import { listValuesJob } from "./list-values-job.ts"; import { PURGE_MAX_REQUESTS, PURGE_TAGS_PER_REQUEST } from "./purge.ts"; @@ -274,7 +273,6 @@ export function adminSystemJobs( }, listValuesJob(sql, { paid: env.CLOUDHAL_TICK_BUDGET === "paid", log }), oauthClientsPruneJob(sql, log), - canonicalRowsJob(sql, log), ]; if (o.commentsOn) jobs.push({ diff --git a/packages/core-ts/src/errors.ts b/packages/core-ts/src/errors.ts index 9a451c85..f502146b 100644 --- a/packages/core-ts/src/errors.ts +++ b/packages/core-ts/src/errors.ts @@ -231,9 +231,9 @@ export type InvariantCode = | "reference_target_unknown" /** §1: more than 100 reference values in one entry, across fields and repeater items. */ | "too_many_references" - /** §2.1: a reference names a group with no usable member of this site in an allowed collection. */ + /** §2.1: a reference names a group with no usable member of this site in an allowed collection (422). */ | "unknown_entry" - /** §2.1, §2.2: the group's member is in a collection the field does not allow, or a move would leave it so. */ + /** §2.2: a move would leave a group a published entry shows without a member in a collection its field allows. */ | "reference_target_moved" /** §2.2: the delete or group change would strand a reference a published entry shows (`details.referrers`). */ | "entry_referenced" diff --git a/packages/core-ts/src/fields/types.ts b/packages/core-ts/src/fields/types.ts index 2a4bbb62..0fbabb29 100644 --- a/packages/core-ts/src/fields/types.ts +++ b/packages/core-ts/src/fields/types.ts @@ -152,10 +152,11 @@ export type FieldIssueCode = | "invalid_query" /** RFC-0045: more than 100 reference values in one entry. */ | "too_many_references" - /** RFC-0045 §2.1: a reference to a group without a usable member of this site. */ + /** + * RFC-0045 §2.1: a reference to a group without a usable member of this site in one of the + * field's collections (missing, foreign, archived, or only in other collections). + */ | "unknown_entry" - /** RFC-0045 §2.1: the group's members are all in collections the field does not allow. */ - | "reference_target_moved" /** RFC-0045 §2.8: a new reference or repeater field over a key entries already hold. */ | "field_key_in_use"; diff --git a/packages/core-ts/src/index.ts b/packages/core-ts/src/index.ts index 0fa2f947..f4e9fe39 100644 --- a/packages/core-ts/src/index.ts +++ b/packages/core-ts/src/index.ts @@ -79,8 +79,6 @@ export { redirectLocale, splitTarget, } from "./redirects.ts"; -// RFC-0045 §2.4: site rows in canonical form (the anchors compare them as text). -export { type CanonicalRowsResult, canonicalizeSiteRows } from "./references/canonical-rows.ts"; export type { PublishReleaseInput, PublishReleaseResult, diff --git a/packages/core-ts/src/list-values/fingerprint.ts b/packages/core-ts/src/list-values/fingerprint.ts index 4b94d668..b3f29347 100644 --- a/packages/core-ts/src/list-values/fingerprint.ts +++ b/packages/core-ts/src/list-values/fingerprint.ts @@ -61,7 +61,9 @@ export async function listFingerprint(site: SiteFields, collection: string): Pro }); return sha256Hex( canonicalJson({ - format: 1, + // 2: RFC-0045 OQ 14, `resolveFields` no longer lends a relinked original's values, so rows + // built under format 1 rebuild once (`lists.backfill`; listings scan until then). + format: 2, sort: settings?.sort ?? "newest", listFields, defs, diff --git a/packages/core-ts/src/local.ts b/packages/core-ts/src/local.ts index e7e4d7db..7b5a770a 100644 --- a/packages/core-ts/src/local.ts +++ b/packages/core-ts/src/local.ts @@ -6,7 +6,6 @@ import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; import { GATES_ENFORCED } from "./availability.ts"; import type { BlockRegistry, PropsSchema } from "./blocks.ts"; import { rebuildListValues } from "./list-values/backfill.ts"; -import { canonicalizeSiteRows } from "./references/canonical-rows.ts"; import { SiteGraphStore, type SiteGraphStoreOptions } from "./store.ts"; export interface LocalStore { @@ -46,8 +45,6 @@ export async function openLocalStore( await migrate(sql, migrations); // RFC-0042 §4: list values for collections listed before migration 0029 or left unfinished. await rebuildListValues(sql); - // RFC-0045 §2.4: site rows stored verbatim by older imports, in canonical form for the anchors. - await canonicalizeSiteRows(sql); const blobs = new FsBlobStore(path.join(dir, "blobs")); // The renderer that serves this store runs in the same process and enforces gates (S15). const store = new SiteGraphStore({ diff --git a/packages/core-ts/src/references/canonical-rows.ts b/packages/core-ts/src/references/canonical-rows.ts deleted file mode 100644 index 1588ac63..00000000 --- a/packages/core-ts/src/references/canonical-rows.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * RFC-0045 §2.4 "Canonical rows, from phase 2": the anchors compare a site row's `collections` - * and definitions as text, so a site row stored verbatim by an import before phase 2 (or by an - * import planned `unchanged`) could make every entry write of a collection conflict. This job - * rewrites such rows in canonical form, one CAS each (`WHERE hash = ?`, the hash unchanged: - * it is computed over canonical JSON already), so it can never race a write (rule 4). - * - * Only site rows are rewritten: they are what the anchors read. The admin Worker runs it as a - * system job, local stores when they open; step 7 runs it once more. Worker-safe. - */ -import type { SqlDriver } from "@cloudhal/storage"; -import { canonicalJson } from "../hash.ts"; - -export interface CanonicalRowsResult { - /** Site rows read. */ - read: number; - /** Rows rewritten (a row that changed meanwhile is skipped: its writer stored it canonically). */ - rewritten: number; -} - -/** Rewrites up to `limit` non-canonical site rows (default 50); idempotent. */ -export async function canonicalizeSiteRows( - sql: SqlDriver, - options: { limit?: number } = {}, -): Promise { - const rows = await sql.all<{ id: string; hash: string; body: string }>( - "SELECT id, hash, body FROM nodes WHERE kind = 'site' ORDER BY id", - ); - let rewritten = 0; - for (const row of rows) { - if (rewritten >= (options.limit ?? 50)) break; - let canonical: string; - try { - canonical = canonicalJson(JSON.parse(row.body)); - } catch { - continue; - } - if (canonical === row.body) continue; - const r = await sql.run( - "UPDATE nodes SET body = ? WHERE id = ? AND kind = 'site' AND hash = ? AND body = ?", - [canonical, row.id, row.hash, row.body], - ); - rewritten += r.changes; - } - return { read: rows.length, rewritten }; -} diff --git a/packages/core-ts/src/references/definition-changes.ts b/packages/core-ts/src/references/definition-changes.ts index a15ca735..9b37e069 100644 --- a/packages/core-ts/src/references/definition-changes.ts +++ b/packages/core-ts/src/references/definition-changes.ts @@ -3,7 +3,8 @@ * stored values, checked by a site write (`planFieldDefinitions`) and kept true by guards on the * site CAS (rule 4); entry writes in between carry the definitions anchor (R3). * - * - **Adding** a reference or repeater field over a key entries already hold (`field_key_in_use`). + * - **Adding** a reference or repeater field over a key entries already hold, or a reference + * sub-field over a key their items hold (`field_key_in_use`). * - **Removing a target collection** while a value's group has a member in it. * - **Removing or retyping a sub-field** that items hold. * - **Turning `multiple` on or off** is refused outright (`change_field_type`), as for selects. @@ -86,6 +87,28 @@ function subFieldRemoved(siteId: string, collection: string, key: string, sub: s } satisfies ReferenceDataChange; } +/** A reference sub-field added to a repeater whose stored items already hold its key. */ +function subFieldAdded( + siteId: string, + at: { collection: string; key: string; sub: string; index: number }, +): ReferenceDataChange { + const { collection, key, sub, index } = at; + return { + collection, + key, + count: { + sql: `SELECT count(*) AS n FROM nodes WHERE ${IN_COLLECTION} + AND EXISTS (SELECT 1 FROM json_each(body, ?) i WHERE json_type(i.value, ?) IS NOT NULL)`, + params: [siteId, collection, `$.fields.${key}`, `$.${sub}`], + }, + issue: (n) => ({ + path: `collections/${collection}/fields/${key}/fields/${index}`, + code: "field_key_in_use", + message: `${entries(n)} of ${collection} already hold ${key}.${sub} in their items; choose another key`, + }), + }; +} + /** The changes of one field kept under its key (same type). */ function compare( siteId: string, @@ -107,6 +130,13 @@ function compare( if (gone.length > 0) data.push(collectionsRemoved(siteId, { collection, key: now.key, field: now.key }, gone)); } + if (was.type === "repeater" && now.type === "repeater") { + const old = new Set(was.fields.map((x) => x.key)); + now.fields.forEach((sub, index) => { + if (sub.type === "reference" && !old.has(sub.key)) + data.push(subFieldAdded(siteId, { collection, key: now.key, sub: sub.key, index })); + }); + } if (was.type === "repeater" && now.type === "repeater") for (const sub of was.fields) { const next = now.fields.find((s) => s.key === sub.key); diff --git a/packages/core-ts/src/references/leave.ts b/packages/core-ts/src/references/leave.ts index 128db6b1..d2f7dc14 100644 --- a/packages/core-ts/src/references/leave.ts +++ b/packages/core-ts/src/references/leave.ts @@ -24,7 +24,7 @@ import { and, type Cond } from "../node-sql.ts"; import { hasReferenceFields } from "./issues.ts"; import { memberGroup } from "./members.ts"; import { definitionOfPath } from "./rows.ts"; -import { collectionsAnchor, otherMember, shownRows } from "./sql.ts"; +import { collectionsAnchor, shownRows, usableMembers } from "./sql.ts"; /** What a delete, move or group change of an entry adds to its batch. */ export interface LeavePlan { @@ -133,13 +133,13 @@ export async function planLeave( const kind = kindOf(input); if (!kind) return undefined; const { stored, next } = input; - const anchor = collectionsAnchor(site); if (convertingTargets(site).has(stored.collection)) throw new InvariantError( "field_operation_running", `a field operation is converting values into references to ${stored.collection}; wait until it finishes`, { collection: stored.collection }, ); + const anchor = await collectionsAnchor(sql, site); if (!hasReferenceFields(site)) return { guard: anchor, diagnose: async () => {} }; const group = memberGroup(stored); const shown = shownRows(site.id, group); @@ -157,12 +157,15 @@ export async function planLeave( next?.status === "archived" || !p.collections.includes(next?.collection ?? ""), ); - const members = needs.map((p) => otherMember(site.id, group, stored.id, p.collections)); + const members = usableMembers(site.id, { + group, + nodeId: stored.id, + needs: needs.map((p) => p.collections), + }); const check = async () => { - for (const m of members) { - const ok = await sql.get<{ ok: number }>(`SELECT ${m.sql} AS ok`, m.params); - if (!ok?.ok) refuse(kind, group, await referrers(sql, site.id, await read())); - } + if (!members) return; + const ok = await sql.get<{ ok: number }>(`SELECT ${members.sql} AS ok`, members.params); + if (!ok?.ok) refuse(kind, group, await referrers(sql, site.id, await read())); }; await check(); const unseen: Cond = { @@ -170,7 +173,7 @@ export async function planLeave( params: [...shown.params, JSON.stringify(pairs.map((p) => p.key))], }; const none: Cond = { sql: `NOT EXISTS (SELECT 1 FROM ${shown.sql})`, params: shown.params }; - const kept = and(...members, unseen) as Cond; + const kept = and(members, unseen) as Cond; return { guard: and(anchor, { sql: `${none.sql} OR (${kept.sql})`, diff --git a/packages/core-ts/src/references/sql.ts b/packages/core-ts/src/references/sql.ts index 101ae212..008b20e4 100644 --- a/packages/core-ts/src/references/sql.ts +++ b/packages/core-ts/src/references/sql.ts @@ -6,15 +6,16 @@ * `nodes_translation_locale` still serve it; * - **"shown by a published entry"**: rows of site S naming G whose holder is published, or that a * published same-site member inherits (`shared` 1, or 2 without its own value of `root_key`); - * - the **collections anchor** and the **definitions anchor**, bound from the `Site` object the - * write planned with (never a second read) and compared with `json(…) IS json(?)` over - * canonical bodies; an absent value binds SQL `NULL`, never the string `'null'`. + * - the **collections anchor** and the **definitions anchor**: the stored text of the fragment, + * read when the write is planned and checked against the `Site` object it planned with, then + * compared as text at batch time (`fragmentAnchor`); an absent value is SQL `NULL`. * * Every probe filters `site_id`. Worker-safe; internal to core-ts. */ import type { Site } from "@cloudhal/core-contracts"; -import type { SqlValue } from "@cloudhal/storage"; -import { canonicalJson } from "../hash.ts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { ConflictError } from "../errors.ts"; +import { sameJson } from "../fields/types.ts"; import type { Cond } from "../node-sql.ts"; import { definitionsPath } from "./rows.ts"; @@ -38,30 +39,52 @@ export function shownRows(siteId: string, group: string): Cond { }; } -/** Bound canonical JSON, or SQL `NULL` for an absent value (`json(NULL) IS json(NULL)` holds). */ -const bound = (value: unknown): SqlValue => (value === undefined ? null : canonicalJson(value)); +type AnchoredSite = Pick; + +/** + * An anchor over one JSON fragment of the site row. The fragment's stored text is read now, with + * the same `json_extract` the batch runs, and must equal the planned value (else the site changed + * since it was planned: `ConflictError`, retry). The batch then compares that exact text, so the + * comparison never depends on the row's key order (a site row an older import stored verbatim + * anchors like any other) and never on a second read the plan did not see. Absent: SQL `NULL`. + */ +async function fragmentAnchor( + sql: SqlDriver, + site: AnchoredSite, + fragment: { path: string; planned: unknown }, +): Promise { + const row = await sql.get<{ rev: number; t: string | null }>( + "SELECT rev, json_extract(body, ?) AS t FROM nodes WHERE id = ? AND kind = 'site'", + [fragment.path, site.id], + ); + const text = row?.t ?? null; + if (!row || !sameJson(text === null ? undefined : JSON.parse(text), fragment.planned)) + throw new ConflictError("site", site.id, site.rev, row?.rev); + return { + sql: "(SELECT json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site') IS ?", + params: [fragment.path, site.id, text], + }; +} /** * §2.2: the site's `collections` are still those the write planned with. Every delete, move and * group change of an entry carries it: it holds every definition and field-operation marker, and * ignores `publishedSnapshotId`, theme, menus and other settings. */ -export function collectionsAnchor(site: Pick): Cond { - return { - sql: "json((SELECT json_extract(body, '$.collections') FROM nodes WHERE id = ? AND kind = 'site')) IS json(?)", - params: [site.id, bound(site.collections)], - }; +export function collectionsAnchor(sql: SqlDriver, site: AnchoredSite): Promise { + return fragmentAnchor(sql, site, { path: "$.collections", planned: site.collections }); } /** §2.1: one collection's definitions are still those the entry write planned with. */ export function definitionsAnchor( - site: Pick, + sql: SqlDriver, + site: AnchoredSite, collection: string, -): Cond { - return { - sql: "json((SELECT json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site')) IS json(?)", - params: [definitionsPath(collection), site.id, bound(site.collections?.[collection]?.fields)], - }; +): Promise { + return fragmentAnchor(sql, site, { + path: definitionsPath(collection), + planned: site.collections?.[collection]?.fields, + }); } /** One target-guard pair: group, allowed collections, and whether the value is newly chosen. */ @@ -89,20 +112,34 @@ export function targetGuard(siteId: string, pairs: readonly TargetPair[]): Cond } /** - * Another member of `group` than `nodeId` in this site, not archived, in one of `collections` - * (the delete, move and group-change rules' "usable member"). + * For every list in `needs`, another member of `group` than `nodeId` in this site, not archived, + * in one of that list's collections: the delete, move and group-change rules' "usable member", + * checked per pair (deviation 1). One JSON parameter, deduplicated by collection list, so the + * statement binds six parameters however many pairs block (D1: 100 per statement). */ -export function otherMember( +export function usableMembers( siteId: string, - group: string, - nodeId: string, - collections: readonly string[], -): Cond { + input: { group: string; nodeId: string; needs: readonly (readonly string[])[] }, +): Cond | undefined { + const lists = new Map(); + for (const c of input.needs) { + const sorted = [...new Set(c)].sort(); + lists.set(JSON.stringify(sorted), { c: sorted }); + } + if (lists.size === 0) return undefined; return { - sql: `EXISTS (SELECT 1 FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' AND n.id <> ? - AND ${memberSql("n", "?")} - AND json_extract(n.body, '$.status') <> 'archived' - AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(?)))`, - params: [siteId, nodeId, group, group, JSON.stringify(collections)], + sql: `(SELECT count(*) FROM json_each(?) q WHERE EXISTS ( + SELECT 1 FROM nodes n WHERE n.site_id = ? AND n.kind = 'contentEntry' AND n.id <> ? + AND ${memberSql("n", "?")} + AND json_extract(n.body, '$.status') <> 'archived' + AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(json_extract(q.value, '$.c'))))) = ?`, + params: [ + JSON.stringify([...lists.values()]), + siteId, + input.nodeId, + input.group, + input.group, + lists.size, + ], }; } diff --git a/packages/core-ts/src/references/writer.ts b/packages/core-ts/src/references/writer.ts index 819f0b13..c6e6f55a 100644 --- a/packages/core-ts/src/references/writer.ts +++ b/packages/core-ts/src/references/writer.ts @@ -7,13 +7,14 @@ * own reference value when the entry becomes published, each group has a member in this site in * an allowed collection, not archived when newly chosen. A newly chosen target also meets * `where`, and no value names the entry's own group (checked when planned). - * - **The definitions anchor**, on every write that changes `fields` or `collection`: the - * collection's definitions as planned. + * - **The definitions anchor**: the collection's definitions as planned, on every write in a site + * with reference fields, and on writes that change `fields` or `collection` elsewhere. * - **The rows**, anchored on the entry's new hash: an anchored `DELETE`, then one plain * `INSERT … SELECT` that reads `shared` from the site row as it is when the batch runs, by key * or `migration.fromKey` (the exact key wins; an aggregate, not a correlated `ORDER BY`, which - * workerd's SQLite does not resolve). A key with no definition then yields `NULL` and fails the batch - * (`isSharedNullViolation` → `ConflictError`); it is never `OR IGNORE`. + * workerd's SQLite does not resolve). A key with no reference or repeater definition (absent, or + * retyped by a concurrent import) yields `NULL` and fails the batch (`isSharedNullViolation` → + * `ConflictError`); it is never `OR IGNORE`. */ import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; import type { SqlDriver, SqlStatement } from "@cloudhal/storage"; @@ -124,7 +125,7 @@ export interface EntryRefsPlan { readonly guard: Cond | undefined; /** The `entry_refs` rows, anchored on the entry's new hash. */ statements(hash: string): SqlStatement[]; - /** After a write that matched no row: `unknown_entry` / `reference_target_moved` if a target went. */ + /** After a write that matched no row: `unknown_entry` if a target went or moved away. */ diagnose(): Promise; } @@ -161,11 +162,13 @@ function useIssue(site: Site, use: Use, members: readonly MemberRow[]): FieldIss const allowed = of.filter((m) => use.collections.includes(m.collection)); const usable = allowed.filter((m) => !use.fresh || m.status !== "archived"); if (usable.length === 0) { + // A write naming an entry outside the field's collections: a value to fix (422), not the + // move race's 409 `reference_target_moved`. if (of.length > 0 && allowed.length === 0) return { path: use.path, - code: "reference_target_moved", - message: `${use.groupId} has no entry in ${use.collections.join(", ")} of this site`, + code: "unknown_entry", + message: `${use.groupId} has no entry in ${use.collections.join(", ")} of this site; pick one from those collections`, }; return { path: use.path, @@ -217,7 +220,7 @@ function useIssues(site: Site, draft: ContentEntry, uses: readonly Use[], member } /** The rows statements for `draft` (an anchored `DELETE`, then the plain `INSERT … SELECT`). */ -function rowStatements(site: Site, draft: ContentEntry, hash: string): SqlStatement[] { +export function entryRefStatements(site: Site, draft: ContentEntry, hash: string): SqlStatement[] { const rows = entryRefRows(fieldDefinitions(site, draft.collection), draft.fields); const out: SqlStatement[] = [ { @@ -236,6 +239,7 @@ function rowStatements(site: Site, draft: ContentEntry, hash: string): SqlStatem THEN CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END END)) FROM nodes s, json_each(json_extract(s.body, ?)) d WHERE s.id = ? AND s.kind = 'site' + AND json_extract(d.value, '$.type') IN ('reference', 'repeater') AND (json_extract(d.value, '$.key') = json_extract(p.value, '$.k') OR json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k'))), ? @@ -260,8 +264,12 @@ const changesFields = (draft: ContentEntry, stored: ContentEntry | undefined) => !stored || stored.collection !== draft.collection || !sameJson(stored.fields, draft.fields); /** - * Plans the reference part of an entry write; throws the plan-time refusals. Sites without - * reference fields add only the definitions anchor (to keep `field_key_in_use` true, R3). + * Plans the reference part of an entry write; throws the plan-time refusals. + * + * The definitions anchor: in a site with reference fields, on every write, since each one + * rewrites its rows and may check targets under the planned definitions (a status-only publish + * included, R7 under a concurrent narrowing). In other sites only on writes that change `fields` + * or `collection`, to keep `field_key_in_use` true (R3). */ export async function planEntryRefs( sql: SqlDriver, @@ -269,11 +277,12 @@ export async function planEntryRefs( draft: ContentEntry, stored: ContentEntry | undefined, ): Promise { - const anchor = changesFields(draft, stored) - ? definitionsAnchor(site, draft.collection) - : undefined; - if (!hasReferenceFields(site)) - return { guard: anchor, statements: () => [], diagnose: async () => {} }; + const references = hasReferenceFields(site); + const anchor = + references || changesFields(draft, stored) + ? await definitionsAnchor(sql, site, draft.collection) + : undefined; + if (!references) return { guard: anchor, statements: () => [], diagnose: async () => {} }; const defs = fieldDefinitions(site, draft.collection); const uses = guardedUses(defs, draft, stored); const groups = [...new Set(uses.map((u) => u.groupId))]; @@ -287,7 +296,7 @@ export async function planEntryRefs( } return { guard: and(anchor, targetGuard(site.id, [...pairs.values()])), - statements: (hash) => rowStatements(site, draft, hash), + statements: (hash) => entryRefStatements(site, draft, hash), diagnose: check, }; } diff --git a/packages/core-ts/src/testing-references.ts b/packages/core-ts/src/testing-references.ts index 63c82da2..d5792adf 100644 --- a/packages/core-ts/src/testing-references.ts +++ b/packages/core-ts/src/testing-references.ts @@ -243,7 +243,8 @@ export function referencesSuite(ctx: SuiteContext): void { expect(await code(set({ venue: "entry_01HZX8K3QMISSING" }))).toBe("unknown_entry"); // another site's id is refused exactly like one that does not exist expect(await code(set({ venue: foreign.id }))).toBe("unknown_entry"); - expect(await code(set({ venue: person1.id }))).toBe("reference_target_moved"); + // an entry outside the field's collections is a value to fix (422), not the move race's 409 + expect(await code(set({ venue: person1.id }))).toBe("unknown_entry"); expect(await code(set({ venue: archived.id }))).toBe("unknown_entry"); expect(await code(set({ speakers: [{ person: event.id }] }))).toBe("invalid_fields"); expect(await rows(event.id)).toEqual([]); @@ -443,6 +444,107 @@ export function referencesSuite(ctx: SuiteContext): void { }); }); + describe("review of PR #205", () => { + /** Reverses every object's key order: a site row an older import stored verbatim. */ + const verbatim = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(verbatim); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value) + .reverse() + .map(([k, v]) => [k, verbatim(v)]), + ); + }; + + it("a verbatim (non-canonical) site row without reference fields takes writes, deletes and relinks", async () => { + const BLURB: FieldDefinition = { + key: "blurb", + type: "text", + labels: L("Blurb"), + localized: false, + }; + const site = await world({ notes: { fields: [BLURB] }, places: {} }); + const fresh0 = (await store().getSite(site.id)) as Site; + await ctx + .sql() + .run("UPDATE nodes SET body = ? WHERE id = ?", [ + JSON.stringify(verbatim(fresh0)), + site.id, + ]); + const stored = await ctx + .sql() + .get<{ body: string }>("SELECT body FROM nodes WHERE id = ?", [site.id]); + expect(stored?.body).not.toBe(canonicalJson(fresh0)); + const note = await entry(site, "notes", { fields: { blurb: "a" } }); + expect(await code(update(note, { fields: { blurb: "b" } }))).toBe("ok"); + const other = await entry(site, "notes", { locale: "tr", slug: "other" }); + const relink = { + translation: { + group: note.id, + method: "human" as const, + source: await sourceOf(await fresh(note)), + }, + }; + expect(await code(update(other, relink))).toBe("ok"); + expect(await code(update(await fresh(note), { collection: "places" }))).not.toBe( + "conflict", + ); + expect(await code(store().deleteContentEntry(await fresh(other)))).toBe("ok"); + }); + + it("20 blocking pairs bind a fixed number of parameters (delete and group change)", async () => { + const WHO: FieldDefinition = { + key: "who", + type: "reference", + labels: L("Who"), + localized: false, + collections: ["people"], + }; + const many: NonNullable = { people: {} }; + for (let i = 0; i < 22; i++) many[`c${i}`] = { fields: [WHO] }; + const site = await world(many); + const ada = await entry(site, "people"); + const adaTr = await translate(ada); + for (let i = 0; i < 22; i++) + await entry(site, `c${i}`, { status: "published", fields: { who: ada.id } }); + // the original's translation leaves the group: the original still holds it + const own = { translation: { group: adaTr.id, method: "original" as const } }; + expect(await code(update(adaTr, own))).toBe("ok"); + // the last member: refused as entry_referenced, never a statement-limit error + expect(await code(store().deleteContentEntry(await fresh(ada)))).toBe("entry_referenced"); + }); + + it("adding a reference sub-field over a key items already hold is field_key_in_use", async () => { + const ROLES: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [{ key: "talk", type: "text", labels: L("Talk") }], + }; + const site = await world({ events: { fields: [ROLES] }, people: {} }); + const event = await entry(site, "events", { fields: { speakers: [{ talk: "x" }] } }); + // items as an import could hold them: a key the repeater does not define (yet) + await ctx + .sql() + .run( + "UPDATE nodes SET body = json_set(body, '$.fields.speakers', json(?)) WHERE id = ?", + [JSON.stringify([{ talk: "x", person: "entry_01HZX8K3QPERS" }]), event.id], + ); + const s = (await store().getSite(site.id)) as Site; + const withPerson = withFields(s, "events", [ + { + ...ROLES, + fields: [ + ...(ROLES as { fields: unknown[] }).fields, + { key: "person", type: "reference", labels: L("Person"), collections: ["people"] }, + ], + } as never, + ]); + expect(await code(store().updateSite(withPerson))).toBe("field_key_in_use"); + }); + }); + describe("races (§2.3): the batch that commits second fails, then retries or refuses", () => { it("R1: a referrer write and deleting the target's last member", async () => { const site = await world(); @@ -472,7 +574,7 @@ export function referencesSuite(ctx: SuiteContext): void { await update(await fresh(hall), { collection: "people" }); }); expect(await code(write.updateContentEntry({ ...talk, fields: { room: hall.id } }))).toBe( - "reference_target_moved", + "unknown_entry", ); const hall2 = await entry(site, "places"); const move = racing(async () => { @@ -755,81 +857,143 @@ export function referencesSuite(ctx: SuiteContext): void { expect(await code(created)).toBe("invalid_translation"); }); - it("R16: a title-only write racing rename_field phase A keeps its rows, through fromKey", async () => { + it("R16: a title-only write racing a field operation's phase A fails the definitions anchor", async () => { + // Since the review of PR #205 every write in a site with reference fields carries the + // definitions anchor, so phase A (rename, set_field_localized) fails the write, which + // retries under the new definitions. The batch-time `shared` read (by key or fromKey, + // the exact key first) is tested on its own in references-store.test.ts. const site = await world(); const hall = await entry(site, "places"); const event = await entry(site, "events", { fields: { venue: hall.id } }); const marker = { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }; - const renamed = (s: Site) => - withFields(s, "events", [ - { ...VENUE, key: "place", migration: marker } as never, - SPONSOR, - SPEAKERS, + const phaseA = [ + (s: Site) => + withFields(s, "events", [ + { ...VENUE, key: "place", migration: marker } as never, + SPONSOR, + SPEAKERS, + ]), + (s: Site) => + withFields(s, "events", [ + { + ...VENUE, + localized: true, + migration: { id: "snap_01HZX8K3Q9MG", fromLocalized: false }, + } as never, + SPONSOR, + SPEAKERS, + ]), + ]; + for (const change of phaseA) { + const write = racing(() => rawSite(site.id, change)); + expect( + await code(write.updateContentEntry({ ...(await fresh(event)), title: "Raced" })), + ).toBe("conflict"); + expect(await rows(event.id)).toEqual([ + { + target_id: hall.id, + field: "venue", + root_key: "venue", + shared: 1, + source_status: "draft", + }, ]); - await racing(() => rawSite(site.id, renamed)).updateContentEntry({ - ...event, - title: "Renamed", - }); - expect(await rows(event.id)).toEqual([ - { - target_id: hall.id, - field: "venue", - root_key: "venue", - shared: 1, - source_status: "draft", - }, - ]); + await rawSite(site.id, (s) => withFields(s, "events", [VENUE, SPONSOR, SPEAKERS])); + } }); - it("R16: a new field reusing a running rename's old key: the exact key wins in shared", async () => { + it("R16: a key with no definition at batch time fails the batch, never drops the row", async () => { const site = await world(); const hall = await entry(site, "places"); const event = await entry(site, "events", { fields: { venue: hall.id } }); - const both = (s: Site) => - withFields(s, "events", [ - { - ...VENUE, - key: "place", - migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, - } as never, - { ...VENUE, localized: true }, - SPONSOR, - SPEAKERS, - ]); - await racing(() => rawSite(site.id, both)).updateContentEntry({ ...event, title: "Both" }); - expect((await rows(event.id))[0]?.shared).toBe(2); + const dropped = (s: Site) => withFields(s, "events", [SPONSOR, SPEAKERS]); + const lost = racing(() => rawSite(site.id, dropped)); + expect(await code(lost.updateContentEntry({ ...event, title: "Lost" }))).toBe("conflict"); + expect(await rows(event.id)).toHaveLength(1); + expect((await fresh(event)).title).toBe(event.title); }); - it("R16: a title-only write racing set_field_localized phase A reads the new shared", async () => { + it("#2: a status-only publish racing a narrowing and a move fails the definitions anchor", async () => { const site = await world(); - const hall = await entry(site, "places"); - const event = await entry(site, "events", { fields: { venue: hall.id } }); - const flipped = (s: Site) => + const target = await entry(site, "places"); + const draft = await entry(site, "events", { fields: { venue: target.id } }); + const publish = racing(async () => { + const s = (await store().getSite(site.id)) as Site; + await store().updateSite( + withFields(s, "events", [ + { ...VENUE, collections: ["places"] } as never, + SPONSOR, + SPEAKERS, + ]), + ); + await update(await fresh(target), { collection: "halls" }); + }); + expect(await code(publish.updateContentEntry({ ...draft, status: "published" }))).toBe( + "conflict", + ); + // the retry plans under the narrowed field and refuses the dangling value + expect(await code(update(await fresh(draft), { status: "published" }))).toBe( + "unknown_entry", + ); + }); + + it("R3: a narrowing and a referrer write, both orders", async () => { + const site = await world(); + const hall = await entry(site, "halls"); + const narrowed = (s: Site) => withFields(s, "events", [ - { - ...VENUE, - localized: true, - migration: { id: "snap_01HZX8K3Q9MG", fromLocalized: false }, - } as never, + { ...VENUE, collections: ["places"] } as never, SPONSOR, SPEAKERS, ]); - await racing(() => rawSite(site.id, flipped)).updateContentEntry({ - ...event, - title: "Flip", + // the referrer commits first: the narrowing's count guard fails, and it refuses on diagnosis + const narrowing = racing(async () => { + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + }); + expect(await code(narrowing.updateSite(narrowed(site)))).toBe( + "field_change_needs_operation", + ); + // the narrowing commits first: the referrer's definitions anchor fails, the retry refuses + const other = await world(); + const hall2 = await entry(other, "halls"); + const event = await entry(other, "events"); + const write = racing(async () => { + await store().updateSite(narrowed((await store().getSite(other.id)) as Site)); }); - expect((await rows(event.id))[0]?.shared).toBe(2); + expect( + await code(write.updateContentEntry({ ...event, fields: { venue: hall2.id } })), + ).toBe("conflict"); + expect(await code(update(await fresh(event), { fields: { venue: hall2.id } }))).toBe( + "unknown_entry", + ); }); - it("R16: a key with no definition at batch time fails the batch, never drops the row", async () => { + it("R13 for a move: a field the plan did not see forces a retry", async () => { const site = await world(); - const hall = await entry(site, "places"); - const event = await entry(site, "events", { fields: { venue: hall.id } }); - const dropped = (s: Site) => withFields(s, "events", [SPONSOR, SPEAKERS]); - const lost = racing(() => rawSite(site.id, dropped)); - expect(await code(lost.updateContentEntry({ ...event, title: "Lost" }))).toBe("conflict"); - expect(await rows(event.id)).toHaveLength(1); - expect((await fresh(event)).title).toBe(event.title); + const place = await entry(site, "places"); + await entry(site, "events", { status: "published", fields: { venue: place.id } }); + // venue allows halls, so the move keeps it; talks.room, published meanwhile, does not + const move = racing(async () => { + await entry(site, "talks", { status: "published", fields: { room: place.id } }); + }); + expect(await code(move.updateContentEntry({ ...place, collection: "halls" }))).toBe( + "reference_target_moved", + ); + }); + + it("R13 for a group change: a field the plan did not see forces a retry", async () => { + const site = await world(); + const hall = await entry(site, "halls"); + const member = await translate(hall, { collection: "places" } as never); + await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + // venue keeps the hall original; talks.room (places only) needs the member that leaves + const leave = racing(async () => { + await entry(site, "talks", { status: "published", fields: { room: hall.id } }); + }); + const own = { translation: { group: member.id, method: "original" as const } }; + expect(await code(leave.updateContentEntry({ ...member, ...own }))).toBe( + "entry_referenced", + ); }); it("a foreign archive does not pin another tenant's entries", async () => { diff --git a/packages/core-ts/test/references-store.test.ts b/packages/core-ts/test/references-store.test.ts index 26ddc3ee..4ca41c05 100644 --- a/packages/core-ts/test/references-store.test.ts +++ b/packages/core-ts/test/references-store.test.ts @@ -14,10 +14,10 @@ import { ConflictError } from "../src/errors.ts"; import { canonicalJson } from "../src/hash.ts"; import { REFS_STATEMENT_BYTES } from "../src/media/refs-rebuild.ts"; import { mapUniqueViolation } from "../src/node-sql.ts"; -import { canonicalizeSiteRows } from "../src/references/canonical-rows.ts"; import { entryRefsRebuild } from "../src/references/rebuild.ts"; import { entryRefRows } from "../src/references/rows.ts"; import { collectionsAnchor, definitionsAnchor } from "../src/references/sql.ts"; +import { entryRefStatements } from "../src/references/writer.ts"; const FIXTURES = join(import.meta.dirname, "../../core-contracts/fixtures/valid/site"); const L = (label: string) => ({ en: { label } }); @@ -28,33 +28,45 @@ async function db() { return sql; } -async function holds(site: Site, stored: unknown = site) { +/** Reverses every object's key order: a body as an older import could have stored it verbatim. */ +function verbatim(value: unknown): unknown { + if (Array.isArray(value)) return value.map(verbatim); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value) + .reverse() + .map(([k, v]) => [k, verbatim(v)]), + ); +} + +async function holds(site: Site, text: string = canonicalJson(site)) { const sql = await db(); await sql.run( "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, 'site', ?, 0, 'h', ?, 't', 't')", - [site.id, site.id, site.id, canonicalJson(stored)], + [site.id, site.id, site.id, text], ); - const check = async (c: { sql: string; params: readonly unknown[] }) => - (await sql.get<{ ok: number }>(`SELECT ${c.sql} AS ok`, c.params as never))?.ok; + const check = async (c: Promise<{ sql: string; params: readonly unknown[] }>) => { + const cond = await c; + return (await sql.get<{ ok: number }>(`SELECT ${cond.sql} AS ok`, cond.params as never))?.ok; + }; return { sql, check }; } -describe("the anchors compare canonical bodies as text", () => { +describe("the anchors compare the stored text, read when planned", () => { const sites = readdirSync(FIXTURES) .filter((f) => f.endsWith(".json")) .map((f) => [f, JSON.parse(readFileSync(join(FIXTURES, f), "utf8")) as Site] as const); - it.each(sites)( - "%s: every collection's definitions and the collections round-trip", - async (_, site) => { - const { check } = await holds(site); - expect(await check(collectionsAnchor(site))).toBe(1); + it.each(sites)("%s: canonical and verbatim rows both anchor", async (_, site) => { + for (const text of [canonicalJson(site), JSON.stringify(verbatim(site))]) { + const { sql, check } = await holds(site, text); + expect(await check(collectionsAnchor(sql, site))).toBe(1); for (const key of Object.keys(site.collections ?? {})) - expect(await check(definitionsAnchor(site, key)), key).toBe(1); - // a collection without definitions, and an absent `collections`, bind SQL NULL - expect(await check(definitionsAnchor(site, "nowhere"))).toBe(1); - }, - ); + expect(await check(definitionsAnchor(sql, site, key)), key).toBe(1); + // a collection without definitions, and an absent `collections`, compare SQL NULL + expect(await check(definitionsAnchor(sql, site, "nowhere"))).toBe(1); + } + }); it("fractional and large bounds survive the stored form", async () => { const fields: FieldDefinition[] = [ @@ -70,20 +82,27 @@ describe("the anchors compare canonical bodies as text", () => { ]; const site = { id: "site_01HZX8K3Q9AB", + rev: 0, collections: { things: { fields } }, } as unknown as Site; - const { check } = await holds(site); - expect(await check(definitionsAnchor(site, "things"))).toBe(1); - expect(await check(collectionsAnchor(site))).toBe(1); + const { sql, check } = await holds(site); + expect(await check(definitionsAnchor(sql, site, "things"))).toBe(1); + expect(await check(collectionsAnchor(sql, site))).toBe(1); }); - it("an absent collections binds NULL, a changed one fails", async () => { - const site = { id: "site_01HZX8K3Q9AC" } as Site; - const { check } = await holds(site); - expect(await check(collectionsAnchor(site))).toBe(1); - expect(await check(collectionsAnchor({ ...site, collections: {} }))).toBe(0); - const other = await holds({ ...site, collections: { a: {} } } as Site); - expect(await other.check(collectionsAnchor(site))).toBe(0); + it("a site that differs from the planned one conflicts; one changed after planning fails", async () => { + const site = { id: "site_01HZX8K3Q9AC", rev: 0 } as Site; + const { sql, check } = await holds(site); + expect(await check(collectionsAnchor(sql, site))).toBe(1); + await expect(collectionsAnchor(sql, { ...site, collections: {} })).rejects.toBeInstanceOf( + ConflictError, + ); + const planned = await collectionsAnchor(sql, site); + await sql.run("UPDATE nodes SET body = ? WHERE id = ?", [ + canonicalJson({ ...site, collections: { a: {} } }), + site.id, + ]); + expect(await check(Promise.resolve(planned))).toBe(0); }); }); @@ -170,18 +189,52 @@ describe("the shared NOT NULL mapping", () => { }); }); -describe("canonical site rows (§2.4)", () => { - it("rewrites a verbatim site row once, under its hash, and leaves canonical ones alone", async () => { +describe("the rows INSERT reads shared from the site row at batch time", () => { + const SITE = "site_01HZX8K3Q9SH"; + const ENTRY = "entry_01HZX8K3Q9EN"; + const ref = (key: string, localized: boolean, extra: object = {}) => ({ + key, + type: "reference", + labels: L(key), + localized, + collections: ["places"], + ...extra, + }); + const planned = { id: SITE, rev: 0, collections: { events: { fields: [ref("venue", false)] } } }; + const entry = { + id: ENTRY, + siteId: SITE, + collection: "events", + status: "draft", + fields: { venue: "entry_01HZX8K3QHALL" }, + }; + + async function run(storedFields: unknown[]) { const sql = await db(); - const site = { id: "site_01HZX8K3Q9CN", rev: 3, collections: { b: {}, a: { fields: [] } } }; - const verbatim = JSON.stringify(site); - await sql.run( - "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, 'site', ?, 3, 'h', ?, 't', 't')", - [site.id, site.id, site.id, verbatim], - ); - expect(await canonicalizeSiteRows(sql)).toEqual({ read: 1, rewritten: 1 }); - const row = await sql.get<{ body: string; hash: string }>("SELECT body, hash FROM nodes"); - expect(row).toEqual({ body: canonicalJson(site), hash: "h" }); - expect(await canonicalizeSiteRows(sql)).toEqual({ read: 1, rewritten: 0 }); + const site = { ...planned, collections: { events: { fields: storedFields } } }; + for (const [id, kind, hash, body] of [ + [SITE, "site", "s", site], + [ENTRY, "contentEntry", "eh", entry], + ] as const) + await sql.run( + "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, ?, ?, 0, ?, ?, 't', 't')", + [id, SITE, kind, id, hash, canonicalJson(body)], + ); + await sql.batch(entryRefStatements(planned as never, entry as never, "eh")); + return sql.all<{ shared: number; root_key: string }>("SELECT shared, root_key FROM entry_refs"); + } + + it("matches a running rename's fromKey, and the exact key wins over it", async () => { + const renamed = ref("place", false, { + migration: { id: "snap_01HZX8K3Q9MG", fromKey: "venue" }, + }); + expect(await run([renamed])).toEqual([{ shared: 1, root_key: "venue" }]); + expect(await run([renamed, ref("venue", true)])).toEqual([{ shared: 2, root_key: "venue" }]); + }); + + it("a key retyped or gone by batch time fails the batch (NOT NULL), never inserts", async () => { + const text = { key: "venue", type: "text", labels: L("venue"), localized: false }; + await expect(run([text])).rejects.toThrow(/NOT NULL constraint failed: entry_refs\.shared/); + await expect(run([])).rejects.toThrow(/NOT NULL constraint failed: entry_refs\.shared/); }); }); From a4862f34c53d77075955bab8caaf8e75e3b06c07 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:57:54 +0300 Subject: [PATCH 5/6] docs: ADR-0046 and RFC status say what 2a ships (PR #205 review #17) ADR-0046 names field_operation_unsupported, the referrer-publish refusal, the text anchors, the untested R9 and R14 and what moves to 2b; the RFC status line says step 2 is split. The changeset drops the removed job and adds the list-values rebuild. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/reference-store-integrity.md | 6 ++---- docs/adr/0046-references-and-repeaters.md | 12 ++++++------ ...5-collections-stage-4-references-and-repeaters.md | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.changeset/reference-store-integrity.md b/.changeset/reference-store-integrity.md index 7503644a..527d941a 100644 --- a/.changeset/reference-store-integrity.md +++ b/.changeset/reference-store-integrity.md @@ -2,7 +2,6 @@ "@cloudhal/storage": minor "@cloudhal/core-ts": minor "@cloudhal/api": patch -"@cloudhal/cloudflare": patch "@cloudhal/mcp-server": patch --- @@ -11,13 +10,12 @@ Reference fields get their index and integrity rules (RFC-0045 §2, rollout step - **Storage.** Migration `0033_entry_refs` adds `entry_refs`, one row per stored reference value, removed with its site. - **Store.** Entry writes keep the rows in their own batch and check their targets at batch time (`unknown_entry`, `reference_target_moved`). Deletes, moves and group changes are refused while a published entry shows a reference to the group's last usable member (`entry_referenced`, `reference_target_moved`, `field_operation_running`). Imports rebuild the rows beside a written node set. Field operations naming a reference or repeater field answer `field_operation_unsupported`. Definition changes that would strand values are refused (`field_key_in_use`, removed target collections or sub-fields, `multiple`). - **API and MCP.** The new refusals answer 409 or 422, with MCP hints. -- **Cloudflare.** The hourly `nodes.canonical` job rewrites site rows that older imports stored verbatim, so the anchors compare equal text; local stores do the same when they open. **Behaviour changes on existing sites** (RFC-0045 decision 3): - relinking a node that other nodes use as their translation group is refused (`translation_group_in_use`, 409); -- `resolveFields` and the store read an original's non-localized values only when the original's own group is its id: a node relinked into another group lends them to nobody (OQ 14); +- `resolveFields` and the store read an original's non-localized values only when the original's own group is its id: a node relinked into another group lends them to nobody (OQ 14). The list-values fingerprint format moves to 2, so every listed collection's `entry_list_values` rebuild once (`lists.backfill`); listings scan until they are ready; - a change of `translation.group` alone gets the same source-group check as a source change, and a write joining a group checks that its node is still that group's original; - imports store node bodies in canonical form (the hash is unchanged); -- deletes, moves and group changes of entries carry the collections anchor, and entry writes that change `fields` or `collection` carry the definitions anchor: a concurrent change of the collections' settings makes them answer 409, and a retry succeeds; +- deletes, moves and group changes of entries carry the collections anchor, and entry writes that change `fields` or `collection` carry the definitions anchor: a concurrent change of the collections' settings makes them answer 409, and a retry succeeds. The anchors compare the stored text read when the write is planned, so site rows stored with any key order work; - field operations that would turn a field into a reference or repeater answer `field_operation_unsupported` (409) instead of `fields_unsupported`. diff --git a/docs/adr/0046-references-and-repeaters.md b/docs/adr/0046-references-and-repeaters.md index 4e7e08a6..14661b21 100644 --- a/docs/adr/0046-references-and-repeaters.md +++ b/docs/adr/0046-references-and-repeaters.md @@ -1,6 +1,6 @@ # ADR-0046: Collections stage 4: reference fields by translation group, guarded by an `entry_refs` index; repeaters of flat items - Status: Accepted -- Date: 2026-11-11 (decided by Halil, RFC-0045 accepted 2026-11-10; lands with rollout step 2) +- Date: 2026-11-11 (decided by Halil, RFC-0045 accepted 2026-11-10; lands with rollout step 2a) - Related: RFC-0045, RFC-0041 / ADR-0043, RFC-0042 / ADR-0044, RFC-0033 / ADR-0035, RFC-0004 (`asset_refs`), RFC-0008 / ADR-0023 (releases), #114, #71 - Sources: [Sanity reference type](https://www.sanity.io/docs/studio/reference-type) · [Payload relationship field](https://payloadcms.com/docs/fields/relationship) · [Cloudflare D1 limits](https://developers.cloudflare.com/d1/platform/limits/) · [SQLite JSON functions](https://www.sqlite.org/json1.html) @@ -8,14 +8,14 @@ Collections have typed fields (ADR-0043), but a venue, a speaker or a brand is still plain text retyped on every entry, and a recipe's ingredients are lines of one text field. Two field types fix that, and each needs integrity rules that hold under D1, which has only atomic batches (AGENTS.md rule 4). ## Decision -1. **Two types.** `reference` (one to five target collections of the same site, one target or up to 50) and `repeater` (up to 50 items of up to ten flat scalar sub-fields; no nesting). Both are refused on every write path (`fields_unsupported`) until rollout step 7, when the site Worker reports `fields: 2`. +1. **Two types.** `reference` (one to five target collections of the same site, one target or up to 50) and `repeater` (up to 50 items of up to ten flat scalar sub-fields; no nesting). Both are refused on every write path until rollout step 7, when the site Worker reports `fields: 2`: definitions with `fields_unsupported`, field operations that name them with `field_operation_unsupported` (until step 4). 2. **A reference stores a translation group id**, `COALESCE(translation.group, id)`, the one identity shared by every language version in every site. It resolves only through members of the page's own site, in the page's locale, then `localization.fallback`. Membership is that expression everywhere (guards, renderer, purges). 3. **`entry_refs`** holds one row per (holder, field path, stored group id), whatever the target's state, written in the holder's own batch, anchored on its new hash, with a plain `INSERT` (never `OR IGNORE`) that reads `shared` from the site row at batch time. Every guard and lookup filters `site_id`. -4. **Strong where the live site depends.** Only what a published entry shows locks: a delete, a move out of every allowed collection, or a group change of a group's last usable member is refused while a published entry shows a reference to it (`entry_referenced`, `reference_target_moved`). Archiving and unpublishing are always allowed; an unresolved reference renders nothing, and publish warns (`referenceIssues`) but never fails. -5. **Races are closed at batch time**, not by reads: the target guard in the referrer's CAS; the definitions anchor (a collection's definitions as planned) on entry writes that change values; the collections anchor on every delete, move and group change; planned pairs with an unseen-field check; the written-group and dependants guards on translation links (`translation_group_in_use`). The cases R1–R16 of RFC-0045 §2.3 are shared-suite tests. -6. **Rebuilds only beside a written node set** (imports, pulls, pushes, restores), chunked below 90 KB a statement. References are not release dependencies. +4. **Strong where the live site depends.** Only what a published entry shows locks: a delete, a move out of every allowed collection, or a group change of a group's last usable member is refused while a published entry shows a reference to it (`entry_referenced`, `reference_target_moved`). Archiving and unpublishing are always allowed; an unresolved reference renders nothing. Publishing a *site* warns (`referenceIssues`) and never fails; publishing a *referrer* whose own value names a group with no usable member is refused (`unknown_entry`). +5. **Races are closed at batch time**, not by reads: the target guard in the referrer's CAS; the definitions anchor (a collection's definitions as planned) on every entry write in a site with reference fields, and on writes that change values elsewhere; the collections anchor on every delete, move and group change. Both anchors compare the fragment's stored text, read when the write is planned and checked against the planned `Site`, so they never depend on the row's key order; planned pairs with an unseen-field check; the written-group and dependants guards on translation links (`translation_group_in_use`). The cases R1–R16 of RFC-0045 §2.3 are shared-suite tests, except R9 (field-operation rewrites, rollout step 2b; unreachable before step 4) and R14 (accepted, holds). +6. **Rebuilds only beside a written node set** (imports, pulls, pushes, restores), chunked below 90 KB a statement. References are not release dependencies (rollout step 2b). ## Consequences -- Existing sites see the phase-2 behaviour changes listed in RFC-0045's "Decisions" (relink refusal, the `readsOriginal` tightening, canonical import bodies, the anchors' 409s on concurrent settings changes). +- Existing sites see the phase-2 behaviour changes listed in RFC-0045's "Decisions" (relink refusal, the `readsOriginal` tightening with a one-time rebuild of listed collections' `entry_list_values`, canonical import bodies, the anchors' 409s on concurrent settings changes, `field_operation_unsupported`). - Purges read the published and previous snapshots' bodies (rollout step 3); no new cache tags. - Per-sub-field localization, `datetime` and multiple values inside items, and sorting by a referenced entry's field stay out of stage 4 (RFC-0045 open questions 2 and 3, §4). diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index abcb4bba..afbacdff 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1,6 +1,6 @@ # RFC-0045: Collections stage 4: reference fields and repeaters -- Status: Accepted (2026-11-10, by Halil), with the three Owner decisions ratified; see "Decisions (2026-11-10)". ADR-0046 landed with rollout step 2. +- Status: Accepted (2026-11-10, by Halil), with the three Owner decisions ratified; see "Decisions (2026-11-10)". ADR-0046 lands with rollout step 2a; step 2 is split into 2a (store and integrity) and 2b (its consumers), see PR #205. Drafted in PR #193 and revised six times after an independent review and five verification passes; the review log and the implementation watch list are at the end. - Author: Halil (Madraka), drafted with Claude · Date: 2026-11-10 - Related: From b352207b9e380579e0d45ad8d0ca9fa66dfe34cf Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:07:48 +0300 Subject: [PATCH 6/6] docs(rfc): RFC-0045 describes the text anchors and the dropped canonical-rows job (PR #205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.1 and §2.2: the anchors read the fragment's stored text when planned, check it against the planned Site and compare it as text at batch time; the definitions anchor is on every entry write in a site with reference fields. §2.4: the canonical-rows job is dropped and step 7 does not run it; canonical import bodies stay as hygiene. Watch list and review log follow, and an "Amended in 2a" note lists the review's design changes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- ...ctions-stage-4-references-and-repeaters.md | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index afbacdff..e0893f3c 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -308,13 +308,19 @@ The member probe uses `nodes_translation_locale` (migration 0025: `site_id, kind AND json_extract(n.body, '$.collection') IN (SELECT value FROM json_each(json_extract(p.value, '$.c'))) AND (json_extract(p.value, '$.f') = 0 OR json_extract(n.body, '$.status') <> 'archived'))) = ? ``` -- **The definitions anchor**, on every entry write that changes `fields` or `collection`. The writer binds its collection's definitions *as planned*: the canonical JSON of `site.collections[c].fields` from the very `Site` object it planned with (`entries.ts:39-41`), so there is no second read and no gap. +- **The definitions anchor** (amended in 2a, PR #205). It is on **every** entry write in a site with reference fields: each such write rewrites its rows and may check targets under the planned definitions, a status-only publish included. Elsewhere it is on writes that change `fields` or `collection`, to keep `field_key_in_use` true. + - *When the write is planned*, the writer reads the fragment's stored text with the same expression the batch runs: `SELECT rev, json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site'`. + - It parses that text and checks it (`sameJson`) against `site.collections[c].fields` of the very `Site` object it planned with (`entries.ts:39-41`). A mismatch means the site changed after the plan read it, and it is a `ConflictError` (retry). + - *At batch time* it compares that exact text: ```sql - json((SELECT json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site')) IS json(?) + (SELECT json_extract(body, ?) FROM nodes WHERE id = ? AND kind = 'site') IS ? ``` - The path is `$.collections."".fields`, and both sides are normalized by `json()`. `NULL IS NULL` holds for a collection without definitions. A unit test round-trips every fixture definition through the stored form (fractional bounds included). + The path is `$.collections."".fields`. An absent fragment reads and binds SQL `NULL`, so `NULL IS NULL` holds for a collection without definitions. + - What the batch compares is exactly what was read and validated, so there is no unseen second read. + - Key order does not matter: a site row an older import stored verbatim anchors like any other. + - A unit test runs every fixture site, stored canonically and with every key order reversed, fractional bounds included. - Equality needs stored bodies in canonical form. Store writes are canonical (`canonicalJson`, `node-sql.ts:62/91`), but `importSite` inserts a bundle's body text verbatim (`sync-plan.ts:300-301`), and SQLite's `json()` keeps key order. So from phase 2, `importSite` stores `canonicalJson(JSON.parse(body))` for every row. The hash is computed over canonical JSON already, so it does not change. A hand-made archive with unsorted keys therefore cannot leave a collection whose writes always conflict. + `importSite` still stores `canonicalJson(JSON.parse(body))` for every row (the hash is over canonical JSON already, so it does not change), as hygiene only: no anchor depends on it. The first draft of this section compared `json(…) IS json(canonical)`. That needed canonical rows, and a site row stored verbatim would have made every write of the site conflict until it was rewritten. The anchor conflicts only when that collection's definitions change, never on other site edits. The loser retries under the new definitions, where `field_key_in_use` then sees its value. This is RFC-0042's rule that list-value rows are valid only "under the fingerprint they were built with" (`list-values/fingerprint.ts`), applied to writes. - **The rows**, anchored on the entry's new hash: `DELETE FROM entry_refs WHERE source_id = ? AND `, then one plain `INSERT … SELECT … FROM json_each(?) WHERE `, appended to `own` (`entries.ts:110`). Every write of the holder rewrites all its rows, so `source_status` stays current. @@ -354,10 +360,13 @@ A target deleted or moved between check and batch fails the guard, and `diagnose AND r.source_collection || '.' || r.field NOT IN (SELECT value FROM json_each(:P))) ) ``` The second `NOT EXISTS` makes a blocking row from a field the plan did not see, committed in between, fail the batch, so the write retries and re-plans with that field. The first revision's form let it through: `events.venue` allows `[places, halls]` and `talks.room` allows `[places]`; a talk published meanwhile with `room` = G is stranded once the last member in `places` goes. - - **The collections anchor**, `json((SELECT json_extract(body, '$.collections') FROM nodes WHERE id = ? AND kind = 'site')) IS json(?)`, bound with the canonical JSON of `site.collections` of the `Site` object the write planned with. When the planned site has no `collections`, the write binds SQL `NULL`, never the string `'null'` (`json('null') IS NULL` is false), so `NULL IS NULL` holds. + - **The collections anchor** (amended in 2a, PR #205) works like the definitions anchor, over `$.collections`: + - when planned, the stored text of `json_extract(body, '$.collections')` is read and checked with `sameJson` against `site.collections` of the `Site` object the write planned with (a mismatch is a `ConflictError`); + - at batch time, `(SELECT json_extract(body, '$.collections') FROM nodes WHERE id = ? AND kind = 'site') IS ?` compares that exact text; + - a site without `collections` reads and binds SQL `NULL`, never the string `'null'`, so `NULL IS NULL` holds. - It holds every collection's definitions and every field-operation marker, so it covers the referrer fields whose allowed collections the guard used (R4) and the markers `field_operation_running` reads (R12). - Unlike the site hash used in the fourth revision, it ignores `publishedSnapshotId`, theme, menus and other settings, so a publish no longer makes concurrent deletes and moves answer 409. - - Canonical bodies (§2.1, §2.4) keep the comparison live. + - It compares stored text read when the write was planned, so key order never makes it fail (§2.1). These writes are rare, so **every** delete, move and group change of a `contentEntry` carries the collections anchor, whether or not the plan saw blocking rows. The first revisions added it only when rows were seen. That left R12 open: during a text → reference conversion, the values that will become references have no rows yet, so a delete planned before the marker committed carried no anchor and went through. With the anchor always present, the batch fails once the marker lands in `collections`, the retry plans against the marker, and `field_operation_running` refuses it. - A refusal answers `409 entry_referenced`, with up to 20 published referrers (id, title, collection, field) and the total. For a group change, it also names the group. @@ -412,7 +421,9 @@ A target deleted or moved between check and batch fails the guard, and `diagnose - A node whose hash moved since the read matches nothing and is skipped: its writer has already rewritten its rows. - Rows of nodes that no longer exist go with `DELETE FROM entry_refs WHERE site_id = ? AND source_id NOT IN (SELECT id FROM nodes WHERE site_id = ?)`. - Its `shared` is computed in JS from the site row read with the chunk, and each chunk also carries the collections anchor, so a definitions change between read and batch fails the chunk, which is retried. - - **Canonical rows, from phase 2, not step 7.** The anchors ship in phase 2, so phase 2 also ships an admin job that rewrites existing node rows canonically. It is per row and guarded by CAS (`UPDATE nodes SET body = ? WHERE id = ? AND hash = ?`, hash unchanged, rule 4), so it cannot race a write. That covers rows imported verbatim before phase 2, and imports that planned `unchanged` (`sync-plan.ts:257, 295`), so the anchors are live on every collection. Step 7 runs it once more. + - **No canonical-rows job** (amended in 2a, PR #205). Earlier revisions shipped an admin job in phase 2 that rewrote existing rows canonically, and ran it again at step 7, because the anchors compared canonical JSON. + - The anchors now compare the stored text read when the write is planned (§2.1, §2.2), so verbatim rows work. The job is dropped, and step 7 does not run it. + - Canonical import bodies stay as hygiene only. - It is a safety net: the gate (§1) means it should find nothing. **5. Assets inside repeaters.** Each of these needs its own change, because none follows from the others: @@ -1010,7 +1021,10 @@ The riskiest SQL and interplay. Every phase PR's review checks the items its pha 1. **Batch-time guard SQL** (phase 2). - **The target guard.** Membership in its index-friendly form, the per-pair `fresh` flag, and `n.site_id = ?`. - - **The definitions anchor and the collections anchor.** Bound from the planned `Site` and never from a second read, compared with `json(...) IS json(?)` over canonical bodies. + - **The definitions anchor and the collections anchor.** + - The fragment's stored text is read when the write is planned and validated with `sameJson` against the planned `Site` (a mismatch is a `ConflictError`). The batch compares exactly that text with `IS`. + - Key order never matters, and nothing the batch compares was read without that check. + - The definitions anchor is on every entry write in a site with reference fields (amended in 2a, PR #205). - **The delete, move and group-change condition.** The blocking-rows fragment, the planned pairs `:P` with the unseen-field `NOT EXISTS`, and "another member, not archived, in `A`". - **The rows `INSERT`.** It is a plain `INSERT`, never `OR IGNORE` on the writer path; `shared` is read in SQL by `key` or `migration.fromKey`; a `NULL` must fail the batch and map to `ConflictError`. - **The written-group guard.** Present on every group or source change, absent when the group is the node's own id. @@ -1037,7 +1051,7 @@ The riskiest SQL and interplay. Every phase PR's review checks the items its pha - `translation_group_in_use`. - The `readsOriginal`/`originalOf` tightening (OQ 14). - The group check on group-only changes. - - Canonical bodies on import, and the CAS-guarded canonicalization job. + - Canonical bodies on import, as hygiene: the canonicalization job was dropped in 2a, because the anchors compare stored text. - The always-on collections anchor's liveness: watch 409 rates in the #105 benchmark and in agents' retry loops. - Each of these is listed in the changeset. @@ -1078,7 +1092,7 @@ Three verifiers found 34 earlier findings resolved. The unresolved ones and the | Group ids change (`translation.group` is writable), so stored ids escape the guards (blocking, two verifiers) | Leaving a group is guarded like a delete, with the site anchor, and refused while a published entry shows the reference (§1, §2.2, R6, tests). Rewriting referrers and a new immutable id are both rejected (Alternatives). | | Import, push, pull and restore bypass the gate (blocking, two verifiers) | `checkBundle` gains `assertFieldTypes`, covering every row path through `importSite`, with one capability shared with interactive writes (§1). Step 7 runs a one-time per-site rebuild (§2.4, §9). | | The blocking expansion treats every row as inherited (blocking, three verifiers) | Rows carry `shared` and `root_key`, mirroring `resolveFields`. One SQL fragment serves the guards, the live `referencedBy` and the purge planner, so live equals snapshot. The policy is stated explicitly (§2, §4, §8). | -| The definitions anchor is read separately from the planned `Site` (blocking) | The anchor binds the definitions of the very `Site` object the write planned with, compared with `json(...) IS json(?)` (§2.1). | +| The definitions anchor is read separately from the planned `Site` (blocking) | The anchor binds the definitions of the very `Site` object the write planned with (§2.1). *Amended in 2a (PR #205):* the fragment's stored text is read when the write is planned, checked against that `Site` with `sameJson`, and compared as text at batch time, not `json(...) IS json(?)`. | | The move guard anchors the wrong collection, and the delete exception ignores allowed collections | Moves, deletes and group changes anchor on the hash of the planned site when rows exist. The exception counts only usable members in collections every blocking field allows (§2.2, R4, R5). | | No emission path for `cloudhal-ref-` tags; old Workers refuse new prefixes; budget; privacy (blocking) | Tags dropped. The purge planner adds referrers' own node tags through `entry_refs`, covering newly visible targets and changed covers. No new prefix, no header ids, no theme code (§2, Purges). | | Rows depend on membership and are not re-created | Rows are kept for every stored value; the site filter is the isolation (§2 invariant, R11). | @@ -1101,7 +1115,7 @@ Three verifiers found 34 earlier findings resolved. The unresolved ones and the | One-hop purges missed depth-2 cards, rollbacks, release runs and "used by" lists (blocking) | Closure to the render depth (two hops), "used by" pages via the groups changed entries name, index mode for ordinary publishes and snapshot mode for rollbacks, releases (re-planned at run time) and restores. Time gates rely on the gate-index TTL. Bounded by `maxReads`/`maxTags`, with purge-everything as the fallback (§2, Purges). | | `set_field_localized` left `shared` stale; text→reference rendered references without rows (blocking) | Phase A flips `shared` in the site write's batch (§7). Conversions into references refuse deletes, moves and group changes of target entries until phase C (`field_operation_running`, R12). The `rename_field` `root_key` lag only over-blocks (§2). | | The bundle gate accepted when support was absent | An absent capability refuses; `CopySiteOptions.fieldTypes` and the Node constant `FIELDS_SUPPORTED` (§1). | -| Imported bodies with unsorted keys break the anchor | `importSite` stores canonical bodies from phase 2 (§2.1). | +| Imported bodies with unsorted keys break the anchor | `importSite` stores canonical bodies from phase 2 (§2.1). *Amended in 2a (PR #205):* the anchors no longer depend on canonical bodies; they compare the stored text read when the write is planned. | | The `skip` set cannot hold item paths | Changed in `planEntryFields` too (§9 step 2). | | The `unused` asset filter is unscoped | Added to step 0 (§9). | | `null` inside items | Means unset in that item, and the schema descriptions say so (§6). | @@ -1122,7 +1136,7 @@ Three verifiers found 34 earlier findings resolved. The unresolved ones and the | Index-mode purges assumed `to` equals the working copy, but any snapshot can be published (blocking, two verifiers) | Chosen: the hybrid. One method for every path: index rows for entries whose working copy equals `to`, plus a `from`/`to` body scan of the entries whose hash differs, found by one `json_each` comparison. `PublishChange` is unchanged. There are tests for an approved request publishing an older snapshot after the referrer was edited, and for a write between `createSnapshot` and `publish` (§2, Purges point 4). | | `shared` was bound from plan-time definitions on writes the definitions anchor does not cover (blocking) | Chosen: derive it inside the single `INSERT … SELECT` from the site row at batch time. Phase A's `UPDATE` corrects rows written before it (§2.1, R16). | | The group check ran only when `source` changed | It runs, with a guard keyed on the written group, whenever `translation.group` or `translation.source` is set or changed (§2.2, R15). | -| Legacy non-canonical rows | The step-7 job rewrites the site's rows canonically (§2.4). | +| Legacy non-canonical rows | The step-7 job rewrites the site's rows canonically (§2.4). *Superseded in 2a (PR #205):* the anchors compare stored text, so no job is needed. | | The `field-refs.ts:81` skip filter path format | It switches to item paths, with the `planEntryFields` skip set (§9 step 2). | | "Used by" should use resolved values; `referencedBy` cards | Resolved values with the same-site original; `referencedBy` results are cards, and drawing their fields is outside the guarantee (§2, Purges point 3). | | Wording: "usable member", "drafts never lock", "the one accepted exception" | The guard protects a member's existence; a draft locks only through a published translation that inherits its value; R8 and R14 are both accepted (Summary, §2). | @@ -1141,7 +1155,7 @@ Three verifiers found 34 earlier findings resolved. The unresolved ones and the | Purges ignored references shown through in-flight conversions (blocking) | Any field-operation marker on `from` or `to` purges everything (§2, Purges point 4). | | The always-on site anchor covers `publishedSnapshotId`, so every publish causes 409s | The collections anchor, `json(json_extract(body, '$.collections'))`, holds the definitions and the markers. The threat-model line is corrected (§2.2, §8). | | The written-group guard blocks a translation becoming its own original | Omitted when the group is the node's own id (§2.2). | -| The canonical rewrite needs CAS and must start with the anchors | A CAS `UPDATE … WHERE hash = ?` job ships in phase 2, and step 7 runs it again (§2.4). | +| The canonical rewrite needs CAS and must start with the anchors | A CAS `UPDATE … WHERE hash = ?` job ships in phase 2, and step 7 runs it again (§2.4). *Superseded in 2a (PR #205):* the job is dropped. | | Skip filter exact match misses list- and item-level issues | Prefix match (§9 step 2). | | A large `json_each(:toEntryRefs)` parameter | Moot: purges no longer compare against the working copy. | | A rebuild during a rename drops rows | `entryRefsRebuild` reads through `migration.fromKey` (§2.4). | @@ -1170,3 +1184,19 @@ Three verifiers found 34 earlier findings resolved. The unresolved ones and the ### Acceptance (2026-11-10) The owner accepted the RFC with the three decisions ratified ("Kabul ediyorum"). "Owner decisions" became "Decisions (2026-11-10)", and [#196](https://github.com/cloudhal-dev/cloudhal/issues/196) is linked from step 0. + +### Amended in 2a (PR #205, 2026-11-11) + +Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumers. The review of 2a changed these parts of the design, and the sections above now say so: + +- **The anchors (§2.1, §2.2).** + - Both read the fragment's stored text with `json_extract` when the write is planned, check it with `sameJson` against the planned `Site`, and compare that exact text at batch time. + - Key order does not matter, and no canonical form is needed. + - In a site with reference fields, the definitions anchor is on every entry write. +- **No canonical-rows job (§2.4).** It is dropped, and step 7 does not run it. Canonical import bodies stay as hygiene. +- **Other changes, recorded as deviations in PR #205 and in ADR-0046:** + - usable members are checked per pair, with one `json_each` parameter; + - a referrer write naming an entry outside the field's collections is refused as `unknown_entry` (422), and `reference_target_moved` (409) is only for moves; + - a reference sub-field added over a key that items already hold is `field_key_in_use`; + - the `shared` subquery matches only reference and repeater definitions, as an aggregate, not a correlated `ORDER BY`, which workerd's SQLite does not resolve; + - the list-values fingerprint format is 2, so listed collections rebuild once after the `readsOriginal` tightening.