diff --git a/.changeset/typed-fields-operations.md b/.changeset/typed-fields-operations.md new file mode 100644 index 00000000..eeea3b51 --- /dev/null +++ b/.changeset/typed-fields-operations.md @@ -0,0 +1,13 @@ +--- +"@cloudhal/core-ts": minor +"@cloudhal/api": minor +"@cloudhal/cli": minor +--- + +Typed custom fields, step 3 (RFC-0041 §6 / ADR-0043): field operations. + +- **`runFieldOperation` / `continueFieldOperation` / `revertFieldOperation` / `fieldOperationStatus`** in `@cloudhal/core-ts` run `rename_field`, `change_field_type`, `set_field_localized` and `remove_field`: a dry run (entries touched, values moved, converted, cleared, removed, unchanged and unconvertible, per locale, with samples), a `before …` snapshot whose id is the operation id, the new definition with its `migration` marker (phase A), entries rewritten in guarded batches of 200 that resume after an interruption (phase B), the marker dropped once nothing is pending (phase C) and an `after` snapshot. Only the draft graph changes. A type change follows a closed conversion table (`convertFieldValue`, exported from `@cloudhal/core-ts/fields`) and refuses values it cannot convert (`field_values_unconvertible`) unless `onInvalid: "clear"`. One operation per collection at a time (`operation_in_progress`). Reverting restores the earlier definition and the entries nobody edited since, and converts edited ones back. +- **Readers stay consistent mid-run:** `resolveFields` converts values still in the old type, and a field being removed reads as gone; a person's entry write during an operation goes through the marker, so edits and the operation converge. +- **Translations:** changing only non-localized field values of an original no longer marks its translations stale (their recorded source hash follows, in the same batch); operations keep fresh translations fresh. +- **API:** `POST /api/sites/:id/collections/:key/fields/operations` (`dryRun`, `?continue=`), `GET …/:operationId`, `POST …/:operationId/revert` (`site:settings` and `content:write`), and the same under `/api/admin` with the deployment token. +- **CLI:** `cloudhal fields rename|retype|localize|remove|continue|revert [--dry-run] [--remote]`; the dry run also lists theme files that read the field directly. diff --git a/packages/api/README.md b/packages/api/README.md index bc011e98..0cfc8851 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -14,6 +14,7 @@ The core routes are below; the feature routes (taxonomy, menus, media, comments, | `/api/sites/:id/pages[/:pageId]` | pages (POST creates and appends to `Site.pageIds` atomically; DELETE needs `?rev=`) | | `/api/sites/:id/entries[/:entryId]` | content entries (`?collection=` filter); `fields` is checked against the collection's typed fields (RFC-0041), and `null` removes a key there | | `/api/sites/:id/collections[/:key]` | the collection registry; `PUT /:key {settings, rev}` (`?dryRun=1`) declares a collection or changes its settings, typed `fields` included; `DELETE /:key?rev=` | +| `/api/sites/:id/collections/:key/fields/operations` | RFC-0041 §6 field operations (`site:settings` and `content:write`): `POST {op, …, dryRun?}` with `op` one of `rename_field {from, to}`, `change_field_type {key, type, options?, multiple?, onInvalid?: "clear"}`, `set_field_localized {key, localized}`, `remove_field {key}` → `{operationId, done, report, rewritten, skipped}` (a dry run writes nothing; `done: false` when the 2,000-entry / 20 s budget ran out, then `POST ?continue=`); `GET /:operationId` → state and pending entries; `POST /:operationId/revert` (again while `done: false`). Only the draft graph changes | | `/api/sites/:id/snapshots` | create (`{label, publish}`), list, `:id/publish`, `:id/rollback`, `:a/diff/:b` | | `/api/agent-keys` | API keys for agents: `POST {name, scopes}` (key shown once), `GET`, `DELETE /:id` — signed-in humans only | | `/mcp` | the Cloudhal MCP server (ADR-0011); session cookie → human, `x-api-key` or `Authorization: Bearer ` → agent limited to the key's scopes (REST answers agent keys with a pointer to `/mcp`) | @@ -32,6 +33,9 @@ Typed fields (RFC-0041) answer with every problem and its path in `details.issue | `field_key_reserved`, `field_key_duplicate`, `field_option_duplicate`, `field_bounds` | 400 | definitions that can never be saved as written | | `too_many_fields` | 422 | over 30 fields per collection, 200 per site or 50 options per select | | `field_change_needs_operation` | 409 | changing a field's type, localization or `multiple`, or dropping a key or option that entries still hold | +| `operation_in_progress` | 409 | another field operation is still rewriting the collection (`details.operationId`) | +| `field_values_unconvertible` | 422 | a type change would drop values the conversion table cannot convert, without `onInvalid: "clear"` (`details.report` is the dry run) | +| `field_operation_incomplete` | 409 | reverting an operation that has not finished, or was already reverted | ```ts import { createApi } from "@cloudhal/api"; diff --git a/packages/api/src/errors.ts b/packages/api/src/errors.ts index ccbffc3e..cdaf86e0 100644 --- a/packages/api/src/errors.ts +++ b/packages/api/src/errors.ts @@ -81,6 +81,10 @@ export const INVARIANT_STATUS: Record` resumes one instead + * - `GET …/collections/:key/fields/operations/:operationId` → its status + * - `POST …/collections/:key/fields/operations/:operationId/revert` → the revert result (call again + * while `done: false`) + * + * Refusals are the store's codes: `operation_in_progress` (409), `field_values_unconvertible` (422, + * with the dry run's report in `details.report`), `field_operation_incomplete` (409), and the + * definition codes (`unknown_field`, `field_key_duplicate`, `field_key_reserved`, …). + */ +import { holds } from "@cloudhal/agent-runtime"; +import type { Actor, FieldOption } from "@cloudhal/core-contracts"; +import { + continueFieldOperation, + FIELD_OPERATIONS, + type FieldOperation, + fieldOperationStatus, + revertFieldOperation, + runFieldOperation, + type SiteGraphStore, +} from "@cloudhal/core-ts"; +import { FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; +import type { Context } from "hono"; +import { Hono } from "hono"; +import type { Env } from "./context.ts"; +import { badRequest, missingScope, notFound } from "./errors.ts"; +import { actorOf } from "./governance.ts"; +import { param } from "./params.ts"; + +type Body = Record; + +const str = (b: Body, name: string): string => { + const v = b[name]; + if (typeof v !== "string" || !v) throw badRequest(`${name} is required`); + return v; +}; + +/** The request body as an operation on `collection`; 400 for a malformed one. */ +export function operationOf(collection: string, b: Body): FieldOperation { + const op = b.op; + if (typeof op !== "string" || !FIELD_OPERATIONS.includes(op as FieldOperation["op"])) + throw badRequest(`op must be one of ${FIELD_OPERATIONS.join(", ")}`); + if (op === "rename_field") return { op, collection, from: str(b, "from"), to: str(b, "to") }; + if (op === "remove_field") return { op, collection, key: str(b, "key") }; + if (op === "set_field_localized") { + if (typeof b.localized !== "boolean") throw badRequest("localized must be a boolean"); + return { op, collection, key: str(b, "key"), localized: b.localized }; + } + const type = str(b, "type"); + if (!FIELD_TYPES.includes(type as FieldType)) + throw badRequest(`type must be one of ${FIELD_TYPES.join(", ")}`); + if (b.onInvalid !== undefined && b.onInvalid !== "clear") + throw badRequest('onInvalid is "clear" or absent'); + if (b.multiple !== undefined && typeof b.multiple !== "boolean") + throw badRequest("multiple must be a boolean"); + if (b.options !== undefined && !Array.isArray(b.options)) + throw badRequest("options must be a list of { value, labels }"); + return { + op: "change_field_type", + collection, + key: str(b, "key"), + type: type as FieldType, + ...(b.options ? { options: b.options as FieldOption[] } : {}), + ...(b.multiple !== undefined ? { multiple: b.multiple as boolean } : {}), + ...(b.onInvalid === "clear" ? { onInvalid: "clear" as const } : {}), + }; +} + +/** The operation must belong to the collection in the path. */ +async function ownStatus(store: SiteGraphStore, siteId: string, key: string, id: string) { + const status = await fieldOperationStatus(store, siteId, id); + if (status.operation.collection !== key) throw notFound("not found: field operation"); + return status; +} + +/** The handlers, for the people's routes and the operator's alike. */ +export function fieldOperationHandlers(actor: (c: Context) => Actor) { + return new Hono() + .post("/:key/fields/operations", async (c) => { + const store = c.get("store"); + const siteId = param(c, "siteId"); + const key = param(c, "key"); + const resume = c.req.query("continue"); + if (resume) { + await ownStatus(store, siteId, key, resume); + return c.json(await continueFieldOperation(store, siteId, resume, { actor: actor(c) })); + } + const b = ((await c.req.json().catch(() => null)) ?? {}) as Body; + if (b.dryRun !== undefined && typeof b.dryRun !== "boolean") + throw badRequest("dryRun must be a boolean"); + const op = operationOf(key, b); + return c.json( + await runFieldOperation(store, siteId, op, { actor: actor(c), dryRun: b.dryRun === true }), + ); + }) + .get("/:key/fields/operations/:operationId", async (c) => + c.json( + await ownStatus( + c.get("store"), + param(c, "siteId"), + param(c, "key"), + param(c, "operationId"), + ), + ), + ) + .post("/:key/fields/operations/:operationId/revert", async (c) => { + const store = c.get("store"); + const siteId = param(c, "siteId"); + const id = param(c, "operationId"); + await ownStatus(store, siteId, param(c, "key"), id); + return c.json(await revertFieldOperation(store, siteId, id, { actor: actor(c) })); + }); +} + +/** + * The people's routes under `/api/sites/:siteId/collections`. `ROUTE_SCOPES` asks `site:settings` + * for the writes; the operations rewrite entries, so `content:write` is checked here too. + */ +export const fieldOperations: Hono = new Hono() + .use("/:key/fields/operations/*", async (c, next) => { + const principal = c.get("principal"); + if (c.req.method === "POST" && principal && !holds(principal.scopes, "content:write")) + throw missingScope("content:write"); + await next(); + }) + .use("/:key/fields/operations", async (c, next) => { + const principal = c.get("principal"); + if (principal && !holds(principal.scopes, "content:write")) throw missingScope("content:write"); + await next(); + }) + .route("/", fieldOperationHandlers(actorOf)); diff --git a/packages/api/src/playground.ts b/packages/api/src/playground.ts index ba882477..f81a8ef7 100644 --- a/packages/api/src/playground.ts +++ b/packages/api/src/playground.ts @@ -114,6 +114,10 @@ export const PLAYGROUND_ALLOW: ReadonlySet = new Set([ `GET ${S}/collections`, `PUT ${S}/collections/:key`, `DELETE ${S}/collections/:key`, + // RFC-0041 §6: field operations rewrite the visitor's own draft entries. + `POST ${S}/collections/:key/fields/operations`, + `GET ${S}/collections/:key/fields/operations/:operationId`, + `POST ${S}/collections/:key/fields/operations/:operationId/revert`, `GET ${S}/pages`, `POST ${S}/pages`, `GET ${S}/pages/:pageId/preview`, diff --git a/packages/api/src/routes/admin.ts b/packages/api/src/routes/admin.ts index f50d6171..741263b4 100644 --- a/packages/api/src/routes/admin.ts +++ b/packages/api/src/routes/admin.ts @@ -35,6 +35,9 @@ * run, then the summary (RFC-0020 open question 7, `cloudhal ai build-eval --remote`); in memory * - `POST /api/admin/sites/:siteId/search/reindex` → rebuilds the search index from the live * snapshot (RFC-0019 §3, `cloudhal search reindex --remote`) + * - `POST /api/admin/sites/:siteId/collections/:key/fields/operations[?continue=]`, `GET + * …/operations/:id` and `POST …/operations/:id/revert`: field operations as the operator + * (RFC-0041 §6, `cloudhal fields … --remote`; the same handlers as Studio's, `../field-operations.ts`) * - `GET /api/admin/sites/:siteId/media-placeholders?all=1` → images without a placeholder, and * `PUT …/media-placeholders/:assetId` ← {rev, lqip?, color?}: `cloudhal media placeholders * --remote` draws them in a local Chrome (RFC-0028 §5) @@ -52,12 +55,13 @@ import type { RenderDeps } from "@cloudhal/mcp-server/tools"; import { Hono } from "hono"; import type { AiTransport } from "../ai/client.ts"; import type { Env } from "../context.ts"; +import { fieldOperationHandlers } from "../field-operations.ts"; import { aiAdminRoutes } from "./admin/ai.ts"; import { analyticsAdminRoutes } from "./admin/analytics.ts"; import { commentsAdminRoutes } from "./admin/comments.ts"; import { ownerAdminRoutes } from "./admin/owner.ts"; import { schedulesAdminRoutes } from "./admin/schedules.ts"; -import { timingSafeEqual } from "./admin/shared.ts"; +import { OPERATOR_ACTOR, timingSafeEqual } from "./admin/shared.ts"; import { siteOpsAdminRoutes } from "./admin/site-ops.ts"; import { siteWorkersAdminRoutes } from "./admin/site-workers.ts"; import { sitesAdminRoutes } from "./admin/sites.ts"; @@ -91,6 +95,10 @@ export function adminRoutes( .route("/", sitesAdminRoutes(deps)) .route("/", siteWorkersAdminRoutes(deps)) .route("/", siteOpsAdminRoutes()) + .route( + "/sites/:siteId/collections", + fieldOperationHandlers(() => OPERATOR_ACTOR), + ) .route("/", syncAdminRoutes(deps)) ); } diff --git a/packages/api/src/routes/collections.ts b/packages/api/src/routes/collections.ts index ebdafeb4..8599d342 100644 --- a/packages/api/src/routes/collections.ts +++ b/packages/api/src/routes/collections.ts @@ -4,12 +4,14 @@ * `PUT /:key { settings, rev }` (site:settings) declares a collection or replaces its settings * through the site update, so the store's overlap and permalink-redirect checks run (`rev` is the * site's: a stale one is 409); `?dryRun=1` runs them and writes nothing. `DELETE /:key?rev=` - * (site:settings) undeclares a collection without entries. + * (site:settings) undeclares a collection without entries. Field operations (RFC-0041 §6) live + * under `/:key/fields/operations` (`../field-operations.ts`). */ import type { CollectionSettings, Site } from "@cloudhal/core-contracts"; import { Hono } from "hono"; import type { Env } from "../context.ts"; import { badRequest, notFound } from "../errors.ts"; +import { fieldOperations } from "../field-operations.ts"; import { param } from "../params.ts"; const KEY = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; @@ -70,4 +72,5 @@ export const collections: Hono = new Hono() if (!site.collections?.[key]) throw notFound("collection not declared"); const next = withCollection({ ...site, rev: revOf(c.req.query("rev")) }, key, undefined); return c.json({ site: await store.updateSite(next) }); - }); + }) + .route("/", fieldOperations); diff --git a/packages/api/src/scopes.ts b/packages/api/src/scopes.ts index 5c2ec321..955179a3 100644 --- a/packages/api/src/scopes.ts +++ b/packages/api/src/scopes.ts @@ -66,6 +66,18 @@ export const ROUTE_SCOPES: readonly (readonly [Method, string, RouteRequirement] ["POST", "/api/admin/sites/:siteId/ai/judge-check", "deploy-token"], // RFC-0019 §3: `cloudhal search reindex --remote` ["POST", "/api/admin/sites/:siteId/search/reindex", "deploy-token"], + // RFC-0041 §6: `cloudhal fields rename|retype|localize|remove|revert --remote` + ["POST", "/api/admin/sites/:siteId/collections/:key/fields/operations", "deploy-token"], + [ + "GET", + "/api/admin/sites/:siteId/collections/:key/fields/operations/:operationId", + "deploy-token", + ], + [ + "POST", + "/api/admin/sites/:siteId/collections/:key/fields/operations/:operationId/revert", + "deploy-token", + ], // RFC-0028 §5: `cloudhal media placeholders --remote` ["GET", "/api/admin/sites/:siteId/media-placeholders", "deploy-token"], ["PUT", "/api/admin/sites/:siteId/media-placeholders/:assetId", "deploy-token"], @@ -124,6 +136,10 @@ export const ROUTE_SCOPES: readonly (readonly [Method, string, RouteRequirement] ["GET", `${S}/collections`, "content:read"], ["PUT", `${S}/collections/:key`, "site:settings"], ["DELETE", `${S}/collections/:key`, "site:settings"], + // RFC-0041 §6: field operations (the handler also asks content:write; they rewrite entries) + ["POST", `${S}/collections/:key/fields/operations`, "site:settings"], + ["GET", `${S}/collections/:key/fields/operations/:operationId`, "content:read"], + ["POST", `${S}/collections/:key/fields/operations/:operationId/revert`, "site:settings"], ["GET", `${S}/pages`, "content:read"], ["POST", `${S}/pages`, "content:write"], ["GET", `${S}/pages/:pageId/preview`, "content:read"], diff --git a/packages/api/src/testing-fields.ts b/packages/api/src/testing-fields.ts index e4f460f9..d3a0890c 100644 --- a/packages/api/src/testing-fields.ts +++ b/packages/api/src/testing-fields.ts @@ -160,5 +160,46 @@ export function describeFieldsApi(label: string, factory: ApiStoreFactory): void expect(made.status, JSON.stringify(made.body)).toBe(201); expect(made.body.fields).toEqual(loose); }); + + it("field operations: dry run, run, status, continue and revert (RFC-0041 §6)", async () => { + expect((await put(EVENTS)).status).toBe(200); + const made = await create({ venue: "Hall", level: "expert" }); + expect(made.status).toBe(201); + const ops = `/api/sites/${siteId}/collections/events/fields/operations`; + const post = (body: Json, query = "") => + call(`${ops}${query}`, { method: "POST", body: JSON.stringify(body) }); + expect((await post({ op: "explode" })).status).toBe(400); + const unknown = await post({ op: "remove_field", key: "colour" }); + expect(unknown.status).toBe(422); + expect(unknown.body.error).toBe("unknown_field"); + const dry = await post({ op: "rename_field", from: "venue", to: "place", dryRun: true }); + expect(dry.status, JSON.stringify(dry.body)).toBe(200); + expect(dry.body).toMatchObject({ dryRun: true, report: { moved: 1, touched: 1 } }); + const run = await post({ op: "rename_field", from: "venue", to: "place" }); + expect(run.status, JSON.stringify(run.body)).toBe(200); + expect(run.body.done).toBe(true); + const id = run.body.operationId as string; + const entry = await call(`/api/sites/${siteId}/entries/${made.body.id}`); + expect(entry.body.fields).toEqual({ place: "Hall", level: "expert" }); + const status = await call(`${ops}/${id}`); + expect(status.body).toMatchObject({ state: "done", pending: 0 }); + expect((await post({}, `?continue=${id}`)).body).toMatchObject({ done: true }); + // A type change that would lose values is refused with the report (422). + const lossy = await post({ op: "change_field_type", key: "level", type: "number" }); + expect(lossy.status).toBe(422); + expect(lossy.body.error).toBe("field_values_unconvertible"); + expect((lossy.body.details as Json).report).toMatchObject({ unconvertible: 1 }); + const back = await call(`${ops}/${id}/revert`, { method: "POST" }); + expect(back.status, JSON.stringify(back.body)).toBe(200); + expect(back.body).toMatchObject({ done: true, restored: 1 }); + expect((await call(`/api/sites/${siteId}/entries/${made.body.id}`)).body.fields).toEqual({ + venue: "Hall", + level: "expert", + }); + const again = await call(`${ops}/${id}/revert`, { method: "POST" }); + expect(again.status).toBe(409); + expect(again.body.error).toBe("field_operation_incomplete"); + expect((await call(`${ops}/snap_doesnotexist`)).status).toBe(404); + }); }); } diff --git a/packages/api/test/admin.test.ts b/packages/api/test/admin.test.ts index f3763490..2979a89a 100644 --- a/packages/api/test/admin.test.ts +++ b/packages/api/test/admin.test.ts @@ -91,6 +91,36 @@ describe("admin search reindex (RFC-0019 §3)", () => { }); }); +describe("admin field operations (RFC-0041 §6)", () => { + it("runs a field operation as the operator behind the deploy token", async () => { + const b = await backend(); + let site = await seedStarterSite(b.store, { name: "Ops", slug: "ops" }); + site = await b.store.updateSite({ + ...site, + collections: { + events: { + fields: [{ key: "venue", type: "text", labels: { en: { label: "V" } }, localized: true }], + }, + }, + }); + const app = createApi({ store: b.store, admin: { deployToken: TOKEN, storage: b } }); + const res = await app.request( + `/api/admin/sites/${site.id}/collections/events/fields/operations`, + { + method: "POST", + headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, + body: JSON.stringify({ op: "rename_field", from: "venue", to: "place" }), + }, + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { done: boolean; operationId: string }; + expect(body.done).toBe(true); + const before = await b.store.getSnapshot(body.operationId); + expect(before?.actor.id).toBe("sys_operator01"); + expect((await b.store.getSite(site.id))?.collections?.events?.fields?.[0]?.key).toBe("place"); + }); +}); + describe("admin placeholder backfill (RFC-0028 §5)", () => { it("lists images without a colour and stores what the CLI drew, keeping measured fields", async () => { const b = await backend(); @@ -314,6 +344,9 @@ describe("admin token guard", () => { ["POST", "/api/admin/sites/site_x/ai/build-eval"], ["POST", "/api/admin/sites/site_x/ai/judge-check"], ["POST", "/api/admin/sites/site_x/search/reindex"], + ["POST", "/api/admin/sites/site_x/collections/events/fields/operations"], + ["GET", "/api/admin/sites/site_x/collections/events/fields/operations/snap_x"], + ["POST", "/api/admin/sites/site_x/collections/events/fields/operations/snap_x/revert"], ["GET", "/api/admin/sites/site_x/media-placeholders"], ["PUT", "/api/admin/sites/site_x/media-placeholders/asset_x"], ["GET", "/api/admin/sites/site_x/bundle"], diff --git a/packages/cli/README.md b/packages/cli/README.md index b0d4053b..2427a47d 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -137,6 +137,22 @@ Locally, `cloudhal studio` reads the project's SQLite (`analytics_events`), roll `cloudhal migrate tags-to-terms` moves the deprecated `ContentEntry.tags` of the local site into terms of the `tag` taxonomy. It snapshots the site first (`before tags-to-terms`; its id is the migration id), creates the default `category` and `tag` taxonomies if they are missing (`category` is not required on an existing site), creates one active tag per distinct slug, rewrites entries in batches of 200 and snapshots again (`tags-to-terms`). An interrupted run resumes; a second run does nothing. Nothing goes live until you publish. `--dry-run` reports what would change; `--revert ` restores the tags of entries nobody edited since, then removes the terms and taxonomies the run created that are no longer used. +### Changing typed fields (RFC-0041) + +`cloudhal fields` changes a collection's typed fields together with the values entries hold, which a plain edit of the definitions refuses (`field_change_needs_operation`): + +```bash +cloudhal fields rename events speaker presenter --dry-run # report only; also lists theme files reading fields.speaker +cloudhal fields rename events speaker presenter +cloudhal fields retype events seats number [--clear] # closed conversion table; --clear drops values it cannot convert +cloudhal fields retype events format select --options talk,workshop=Workshop [--multiple] +cloudhal fields localize events notes off # translations' own copies go; the original's value is read +cloudhal fields remove events notes +cloudhal fields revert events +``` + +Each run snapshots the site first (`before rename_field events.speaker`; its id is the operation id), writes the new definition with a `migration` marker so every reader stays consistent, rewrites entries in guarded batches of 200, drops the marker and snapshots again. Translations that were up to date stay up to date. An interrupted run resumes with `cloudhal fields continue `; one operation per collection runs at a time. `--remote` runs it on the deployment with the deployment token. Only the draft graph changes: nothing goes live until you publish. + ### Deploying to Cloudflare (ADR-0012) `cloudhal deploy` uses **your** wrangler login (`npx wrangler login`, or `CLOUDFLARE_API_TOKEN`); Cloudhal never reads those credentials. It creates the D1 database and R2 bucket if needed (with a lifecycle rule that expires the snapshot bundles under `bundles/` after 30 days, RFC-0010), generates the deployment secrets once into `.cloudhal/deploy/secrets.json` (0600 — keep it out of git), builds and deploys the **site Worker** (Astro on demand, edge-cached) and the **admin Worker** (Studio, API, auth, MCP), and on the first deploy pushes your local site. diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 36c9a54e..d7fd9047 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -65,6 +65,9 @@ export const OPTIONS = { project: { type: "string" }, theme: { type: "string" }, confirm: { type: "string" }, + options: { type: "string" }, + multiple: { type: "boolean" }, + clear: { type: "boolean" }, } as const; /** `cloudhal ` → its handler. */ diff --git a/packages/cli/src/cli/commands-content.ts b/packages/cli/src/cli/commands-content.ts index e619665a..b062d873 100644 --- a/packages/cli/src/cli/commands-content.ts +++ b/packages/cli/src/cli/commands-content.ts @@ -4,6 +4,7 @@ import { commentsStatusCli, describeCommentsStatus, } from "../commands/comments.ts"; +import { fieldsCommand } from "../commands/fields.ts"; import { describeMediaImport, mediaGc, mediaImportUrls, mediaVerify } from "../commands/media.ts"; import { describePlaceholders, mediaPlaceholders } from "../commands/media-placeholders.ts"; import { describeMigration, migrateTags } from "../commands/migrate.ts"; @@ -139,6 +140,7 @@ const commentsCommand = subcommands( export const CONTENT_COMMANDS = { migrate: migrateCommand, + fields: fieldsCommand, media: mediaCommand, search: searchCommand, comments: commentsCommand, diff --git a/packages/cli/src/cli/help.ts b/packages/cli/src/cli/help.ts index 1f6f77c3..be7bb068 100644 --- a/packages/cli/src/cli/help.ts +++ b/packages/cli/src/cli/help.ts @@ -37,6 +37,10 @@ Usage: cloudhal migrate tags-to-terms [--dry-run] [--revert ] move entry tags into tag terms (RFC-0002); resumable; --revert undoes a run + cloudhal fields rename | retype [--options a,b] [--multiple] [--clear] | localize on|off | remove | continue|revert [--dry-run] [--remote] + change a collection's typed fields with their values + (RFC-0041): snapshotted, resumable; --dry-run reports + and greps the theme; revert undoes a run cloudhal media import-urls [--dry-run] [--include-markdown] [--allow-host ]… | --revert copy linked images into the media library (RFC-0004); revertible diff --git a/packages/cli/src/commands/fields.ts b/packages/cli/src/commands/fields.ts new file mode 100644 index 00000000..8910069c --- /dev/null +++ b/packages/cli/src/commands/fields.ts @@ -0,0 +1,281 @@ +/** + * `cloudhal fields rename|retype|localize|remove|continue|revert` (RFC-0041 §6): field operations + * on the local site, or with `--remote` on the deployment (deployment token). `--dry-run` prints + * the report and writes nothing; it also greps the project's theme (`src/`) for code that reads + * the field directly (`fields.`), which a rename, type change or removal would break. A run + * snapshots first, rewrites entries in guarded batches and loops until done; `revert ` undoes + * it. Nothing goes live until a person publishes. + */ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import type { FieldOption } from "@cloudhal/core-contracts"; +import { + continueFieldOperation, + type FieldOperation, + type FieldOperationResult, + type FieldRevertResult, + revertFieldOperation, + runFieldOperation, +} from "@cloudhal/core-ts"; +import { FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import { humanizeSlug } from "@cloudhal/core-ts/names"; +import type { CliCommand } from "../cli/types.ts"; +import { remoteFlags } from "../cli/types.ts"; +import { readProjectConfig } from "../config.ts"; +import { brief, deployment } from "./sync.ts"; + +const CLI_ACTOR = { kind: "human", id: "user_localcli0001", name: "cloudhal CLI" } as const; + +export const FIELDS_USAGE = + "usage: cloudhal fields rename | retype [--options a,b] [--multiple] [--clear] | localize on|off | remove | continue|revert [--dry-run] [--remote]"; + +export interface FieldsCliOptions { + readonly projectDir: string; + readonly remote?: boolean; + readonly adminUrl?: string; + readonly fetch?: typeof fetch; +} + +type Request = + | { kind: "run"; op: FieldOperation; dryRun: boolean } + | { kind: "continue" | "revert"; collection: string; id: string }; + +export type FieldsCliResult = + | { kind: "run"; result: FieldOperationResult; theme: string[] } + | { kind: "revert"; result: FieldRevertResult }; + +/** `--options talk,workshop` (or `talk=Talk`) as select options labelled in `locale`. */ +function optionsOf(list: string, locale: string): FieldOption[] { + return list + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + .map((s) => { + const [value, label] = s.split("=", 2) as [string, string | undefined]; + return { value, labels: { [locale]: label?.trim() || humanizeSlug(value) } }; + }); +} + +/** The positionals and flags as a request (throws the usage when they do not fit). */ +export function parseFieldsArgs( + rest: readonly string[], + flags: { dryRun?: boolean; options?: string; multiple?: boolean; clear?: boolean }, + locale = "en", +): Request { + const [action, collection, a, b] = rest; + const need = (...xs: (string | undefined)[]) => { + if (!collection || xs.some((x) => !x)) throw new Error(FIELDS_USAGE); + }; + const dryRun = flags.dryRun === true; + const c = collection as string; + switch (action) { + case "rename": + need(a, b); + return { + kind: "run", + dryRun, + op: { op: "rename_field", collection: c, from: a as string, to: b as string }, + }; + case "remove": + need(a); + return { kind: "run", dryRun, op: { op: "remove_field", collection: c, key: a as string } }; + case "localize": + need(a); + if (b !== "on" && b !== "off") throw new Error(FIELDS_USAGE); + return { + kind: "run", + dryRun, + op: { op: "set_field_localized", collection: c, key: a as string, localized: b === "on" }, + }; + case "retype": { + need(a, b); + if (!FIELD_TYPES.includes(b as FieldType)) + throw new Error(`the type is one of ${FIELD_TYPES.join(", ")}`); + const op: FieldOperation = { + op: "change_field_type", + collection: c, + key: a as string, + type: b as FieldType, + ...(flags.options ? { options: optionsOf(flags.options, locale) } : {}), + ...(b === "select" ? { multiple: flags.multiple === true } : {}), + ...(flags.clear ? { onInvalid: "clear" as const } : {}), + }; + return { kind: "run", dryRun, op }; + } + case "continue": + case "revert": + need(a); + return { kind: action, collection: c, id: a as string }; + default: + throw new Error(FIELDS_USAGE); + } +} + +const SOURCE = /\.(astro|[cm]?[jt]sx?|mdx?)$/; + +/** Theme files under `src/` that read `fields.` directly (RFC-0041 §6 dry run, CLI only). */ +export async function themeReferences(projectDir: string, key: string): Promise { + const pattern = new RegExp( + `fields(?:\\?\\.|\\.)${key}\\b|fields\\[\\s*["'\`]${key}["'\`]\\s*\\]`, + ); + const root = path.join(projectDir, "src"); + const out: string[] = []; + let files: string[]; + try { + files = (await readdir(root, { recursive: true })) as string[]; + } catch { + return out; + } + for (const file of files.sort()) { + if (!SOURCE.test(file)) continue; + const text = await readFile(path.join(root, file), "utf8").catch(() => ""); + if (pattern.test(text)) out.push(path.join("src", file)); + } + return out; +} + +const keyOf = (op: FieldOperation) => (op.op === "rename_field" ? op.from : op.key); + +async function runLocal(options: FieldsCliOptions, req: Request): Promise { + const cfg = await readProjectConfig(options.projectDir); + const local = await openLocalStore(path.join(options.projectDir, cfg.dataDir)); + const actor = { actor: CLI_ACTOR }; + try { + if (req.kind === "revert") { + let result = await revertFieldOperation(local.store, cfg.siteId, req.id, actor); + while (!result.done && result.skipped.length === 0) + result = await revertFieldOperation(local.store, cfg.siteId, req.id, actor); + return { kind: "revert", result }; + } + let result = + req.kind === "run" + ? await runFieldOperation(local.store, cfg.siteId, req.op, { ...actor, dryRun: req.dryRun }) + : await continueFieldOperation(local.store, cfg.siteId, req.id, actor); + while (!result.dryRun && !result.done && result.skipped.length === 0) + result = await continueFieldOperation(local.store, cfg.siteId, result.operationId, actor); + const theme = + req.kind === "run" ? await themeReferences(options.projectDir, keyOf(req.op)) : []; + return { kind: "run", result, theme }; + } finally { + await local.close(); + } +} + +async function runRemote(options: FieldsCliOptions, req: Request): Promise { + const { cfg, adminUrl, token, fetch: f } = await deployment(options); + const collection = req.kind === "run" ? req.op.collection : req.collection; + const base = `/api/admin/sites/${cfg.siteId}/collections/${collection}/fields/operations`; + const call = async (where: string, body?: unknown) => { + const res = await f(new URL(where, adminUrl), { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify(body ?? {}), + }); + const text = await res.text(); + if (!res.ok) throw new Error(`the deployment answered ${res.status}: ${brief(text)}`); + return JSON.parse(text) as FieldOperationResult & FieldRevertResult; + }; + if (req.kind === "revert") { + let result = await call(`${base}/${req.id}/revert`); + while (!result.done && result.skipped.length === 0) + result = await call(`${base}/${req.id}/revert`); + return { kind: "revert", result }; + } + const { + op: _op, + collection: _c, + ...fields + } = req.kind === "run" ? req.op : { op: "", collection: "" }; + let result = + req.kind === "run" + ? await call(base, { op: req.op.op, ...fields, dryRun: req.dryRun }) + : await call(`${base}?continue=${encodeURIComponent(req.id)}`); + while (!result.dryRun && !result.done && result.skipped.length === 0) + result = await call(`${base}?continue=${encodeURIComponent(result.operationId)}`); + const theme = req.kind === "run" ? await themeReferences(options.projectDir, keyOf(req.op)) : []; + return { kind: "run", result, theme }; +} + +/** Runs one `cloudhal fields` request locally or on the deployment. */ +export async function fieldsCli(options: FieldsCliOptions, req: Request): Promise { + return options.remote ? runRemote(options, req) : runLocal(options, req); +} + +/** The site's default language for option labels (the local copy; `en` without one). */ +async function localDefaultLocale(projectDir: string): Promise { + try { + const cfg = await readProjectConfig(projectDir); + const local = await openLocalStore(path.join(projectDir, cfg.dataDir)); + try { + return (await local.store.getSite(cfg.siteId))?.defaultLocale ?? "en"; + } finally { + await local.close(); + } + } catch { + return "en"; + } +} + +const show = (v: unknown) => { + const s = typeof v === "string" ? JSON.stringify(v) : JSON.stringify(v ?? null); + return s.length > 60 ? `${s.slice(0, 59)}…` : s; +}; + +/** Terminal lines for a result. */ +export function describeFields(r: FieldsCliResult): string[] { + if (r.kind === "revert") { + const x = r.result; + return [ + `${x.done ? "reverted" : "reverting"} ${x.operationId}: ${x.restored} entries restored`, + ...(x.edited.length ? [`converted back (edited since): ${x.edited.join(", ")}`] : []), + ...(x.done ? [`snapshot ${x.snapshotId}`] : ["not finished: run the revert again"]), + ]; + } + const { result: x, theme } = r; + const p = x.report; + const verb = x.dryRun ? "would touch" : "touched"; + const lines = [ + `${p.op} ${p.collection}.${p.key}: ${verb} ${p.touched} of ${p.entries} entries (moved ${p.moved}, converted ${p.converted}, cleared ${p.cleared}, removed ${p.removed}, unchanged ${p.unchanged}, unconvertible ${p.unconvertible})`, + ]; + for (const [locale, c] of Object.entries(p.locales)) + lines.push(` ${locale}: ${c.touched} entries, ${c.converted} converted, ${c.cleared} cleared`); + for (const s of p.samples) + lines.push( + ` ${s.entryId} (${s.locale}) ${s.kind}: ${show(s.before)} → ${show(s.after)}${s.reason ? ` (${s.reason})` : ""}`, + ); + if (p.refusal) lines.push(`refused: ${p.refusal.message}`); + if (theme.length) + lines.push(`theme code reads fields.${p.key} directly and needs a change: ${theme.join(", ")}`); + if (x.dryRun) return lines; + if (!x.done) { + lines.push( + `not finished (${x.skipped.length} entries changed meanwhile): cloudhal fields continue ${p.collection} ${x.operationId}`, + ); + return lines; + } + lines.push( + `operation ${x.operationId} done (undo: cloudhal fields revert ${p.collection} ${x.operationId})`, + "nothing is live yet: review the snapshot diff and publish", + ); + return lines; +} + +/** `cloudhal fields …`. */ +export const fieldsCommand: CliCommand = async (rest, values, { projectDir }) => { + const locale = values.options ? await localDefaultLocale(projectDir) : "en"; + const req = parseFieldsArgs( + rest, + { + ...(values["dry-run"] ? { dryRun: true } : {}), + ...(values.options ? { options: values.options } : {}), + ...(values.multiple ? { multiple: true } : {}), + ...(values.clear ? { clear: true } : {}), + }, + locale, + ); + const r = await fieldsCli({ projectDir, ...remoteFlags(values) }, req); + for (const line of describeFields(r)) console.log(line); + if (r.kind === "run" && r.result.report.refusal && r.result.dryRun) return 1; + return r.result.done || (r.kind === "run" && r.result.dryRun) ? 0 : 1; +}; diff --git a/packages/cli/test/fields-cli.test.ts b/packages/cli/test/fields-cli.test.ts new file mode 100644 index 00000000..7cafbd1d --- /dev/null +++ b/packages/cli/test/fields-cli.test.ts @@ -0,0 +1,124 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { main } from "../src/cli.ts"; +import { parseFieldsArgs, themeReferences } from "../src/commands/fields.ts"; +import { writeProjectConfig } from "../src/index.ts"; + +const person = { kind: "human" as const, id: "user_fieldscli01", name: "Owner" }; + +describe("cloudhal fields (RFC-0041 §6)", () => { + let projectDir: string; + let siteId: string; + let entryId: string; + const log = () => vi.spyOn(console, "log").mockImplementation(() => {}); + const lines = (spy: ReturnType) => spy.mock.calls.map((c) => String(c[0])); + + beforeAll(async () => { + projectDir = await mkdtemp(path.join(os.tmpdir(), "cloudhal-fields-")); + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + const site = await local.store.createSite({ + projectId: "proj_fieldscli01", + name: "Events", + slug: "events", + defaultLocale: "en", + locales: ["en"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: { + events: { + fields: [ + { key: "venue", type: "text", labels: { en: { label: "Venue" } }, localized: true }, + ], + }, + }, + }); + siteId = site.id; + entryId = ( + await local.store.createContentEntry({ + siteId, + collection: "events", + slug: "launch", + locale: "en", + status: "draft", + title: "Launch", + body: { format: "markdown", value: "x" }, + fields: { venue: "Hall" }, + tags: [], + author: person, + }) + ).id; + await local.close(); + await writeProjectConfig(projectDir, { siteId, dataDir: ".cloudhal" }); + await mkdir(path.join(projectDir, "src", "layouts"), { recursive: true }); + await writeFile( + path.join(projectDir, "src", "layouts", "Event.astro"), + "

{entry.fields.venue}

\n", + ); + }); + + afterAll(async () => { + await rm(projectDir, { recursive: true, force: true }); + }); + + const fields = async () => { + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + try { + return (await local.store.getContentEntry(entryId))?.fields; + } finally { + await local.close(); + } + }; + + it("parses each action and refuses what does not fit", () => { + expect(parseFieldsArgs(["rename", "events", "a", "b"], {})).toMatchObject({ + kind: "run", + op: { op: "rename_field", from: "a", to: "b" }, + }); + expect( + parseFieldsArgs(["retype", "events", "k", "select"], { options: "talk,panel=Panel talk" }), + ).toMatchObject({ + op: { + type: "select", + multiple: false, + options: [ + { value: "talk", labels: { en: "Talk" } }, + { value: "panel", labels: { en: "Panel talk" } }, + ], + }, + }); + expect(() => parseFieldsArgs(["localize", "events", "k", "maybe"], {})).toThrow(/usage/); + expect(() => parseFieldsArgs(["retype", "events", "k", "colour"], {})).toThrow(/type is one/); + expect(() => parseFieldsArgs(["nope"], {})).toThrow(/usage: cloudhal fields/); + }); + + it("--dry-run reports, greps the theme and writes nothing; a run and its revert", async () => { + expect(await themeReferences(projectDir, "venue")).toEqual([ + path.join("src", "layouts", "Event.astro"), + ]); + const cwd = vi.spyOn(process, "cwd").mockReturnValue(projectDir); + let out = log(); + expect(await main(["fields", "rename", "events", "venue", "place", "--dry-run"])).toBe(0); + expect(lines(out)[0]).toContain("rename_field events.venue: would touch 1 of 1 entries"); + expect(lines(out).join("\n")).toContain("theme code reads fields.venue directly"); + out.mockRestore(); + expect(await fields()).toEqual({ venue: "Hall" }); + + out = log(); + expect(await main(["fields", "rename", "events", "venue", "place"])).toBe(0); + const done = lines(out).find((l) => l.startsWith("operation ")); + out.mockRestore(); + expect(await fields()).toEqual({ place: "Hall" }); + const id = /operation (\S+) done/.exec(done ?? "")?.[1] as string; + + out = log(); + expect(await main(["fields", "revert", "events", id])).toBe(0); + expect(lines(out)[0]).toContain(`reverted ${id}: 1 entries restored`); + out.mockRestore(); + cwd.mockRestore(); + expect(await fields()).toEqual({ venue: "Hall" }); + }); +}); diff --git a/packages/core-ts/src/errors.ts b/packages/core-ts/src/errors.ts index 779ede6c..2aa9c529 100644 --- a/packages/core-ts/src/errors.ts +++ b/packages/core-ts/src/errors.ts @@ -195,7 +195,13 @@ export type InvariantCode = /** More fields than a collection (30) or a site (200) may have, or options than a select (50). */ | "too_many_fields" /** A definition change that could lose or reinterpret stored values; it needs a field operation. */ - | "field_change_needs_operation"; + | "field_change_needs_operation" + /** RFC-0041 §6: another field operation is still rewriting this collection's entries. */ + | "operation_in_progress" + /** RFC-0041 §6: values a type change cannot convert, without `onInvalid: "clear"` (`details.report`). */ + | "field_values_unconvertible" + /** RFC-0041 §6: the field operation has not finished (revert waits for it), or was already reverted. */ + | "field_operation_incomplete"; export class InvariantError extends Error { override readonly name = "InvariantError"; diff --git a/packages/core-ts/src/fields-operations/index.ts b/packages/core-ts/src/fields-operations/index.ts new file mode 100644 index 00000000..ce5febf5 --- /dev/null +++ b/packages/core-ts/src/fields-operations/index.ts @@ -0,0 +1,23 @@ +/** + * RFC-0041 §6 / ADR-0043 §8: field operations — `rename_field`, `change_field_type`, + * `set_field_localized` and `remove_field` — as resumable, snapshotted, revertible runs over the + * draft graph. Exported from `@cloudhal/core-ts`. + */ +export { revertFieldOperation } from "./revert.ts"; +export { continueFieldOperation, fieldOperationStatus, runFieldOperation } from "./run.ts"; +export { + FIELD_MIGRATION, + FIELD_OPERATION_BATCH, + FIELD_OPERATION_BUDGET, + FIELD_OPERATIONS, + type FieldOperation, + type FieldOperationName, + type FieldOperationOptions, + type FieldOperationReport, + type FieldOperationResult, + type FieldOperationSample, + type FieldOperationState, + type FieldOperationStatus, + type FieldRevertResult, + type FieldValueCounts, +} from "./types.ts"; diff --git a/packages/core-ts/src/fields-operations/plan.ts b/packages/core-ts/src/fields-operations/plan.ts new file mode 100644 index 00000000..dbc4c1ff --- /dev/null +++ b/packages/core-ts/src/fields-operations/plan.ts @@ -0,0 +1,227 @@ +/** + * RFC-0041 §6: what an operation asks of the site node. The checks a request must pass, the + * definition it ends with, the marker phase A writes, the SQL predicate of entries it still has + * to rewrite (phase C's `NOT EXISTS` guard), and the site transforms of phases A and C. Pure + * except for the predicates' SQL text. + */ +import type { FieldDefinition, FieldMigration, Site } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { RESERVED_FIELD_KEYS } from "../fields/types.ts"; +import type { Cond } from "../node-sql.ts"; +import { FIELD_OPERATIONS, type FieldOperation, finalKey, operationKey } from "./types.ts"; + +const KEY = /^[a-z][A-Za-z0-9]{0,39}$/; +const COLLECTION = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +function refuse(code: "unknown_field" | "field_key_duplicate" | "field_key_reserved", msg: string) { + return new InvariantError(code, msg, { issues: [] }); +} + +/** The collection's definitions; throws when the collection or the field does not exist. */ +export function currentDefinition( + site: Site, + op: FieldOperation, +): { defs: readonly FieldDefinition[]; def: FieldDefinition; index: number } { + if (!FIELD_OPERATIONS.includes(op.op)) + throw new InvariantError("invalid_fields", `unknown field operation ${String(op.op)}`); + if (!COLLECTION.test(op.collection) || !site.collections?.[op.collection]) + throw new InvariantError( + "collection_undeclared", + `${op.collection} is not a declared collection of this site`, + { collection: op.collection }, + ); + const defs = site.collections[op.collection]?.fields ?? []; + const key = operationKey(op); + const index = defs.findIndex((d) => d.key === key); + const def = defs[index]; + if (!def) + throw refuse( + "unknown_field", + `${key} is not a field of ${op.collection}${defs.length ? `; its fields are ${defs.map((d) => d.key).join(", ")}` : ""}`, + ); + return { defs, def, index }; +} + +/** The field another operation is rewriting in this collection, if any. */ +export function runningMarker(defs: readonly FieldDefinition[]): FieldMigration | undefined { + return defs.find((d) => d.migration)?.migration; +} + +const COMMON = ["key", "type", "labels", "localized", "requiredToPublish"] as const; + +/** The definition `change_field_type` ends with: the shared properties plus the new type's. */ +function retyped(def: FieldDefinition, op: Extract) { + const base: Record = {}; + const was = def as unknown as Record; + for (const k of COMMON) if (was[k] !== undefined) base[k] = was[k]; + base.type = op.type; + if (op.type === "boolean") delete base.requiredToPublish; + const searchable = "searchable" in def ? def.searchable : undefined; + if ((op.type === "text" || op.type === "richText") && searchable !== undefined) + base.searchable = searchable; + if (op.type === "select") { + const options = op.options ?? (def.type === "select" ? def.options : undefined); + if (!options?.length) + throw new InvariantError("invalid_fields", "a select needs its options", { + issues: [{ path: "options", code: "invalid_value", message: "at least one option" }], + }); + base.options = options; + const multiple = op.multiple ?? (def.type === "select" ? def.multiple : undefined); + if (multiple) base.multiple = true; + } + return base as unknown as FieldDefinition; +} + +/** The definition the operation ends with; `null` for `remove_field`. */ +export function targetDefinition(def: FieldDefinition, op: FieldOperation): FieldDefinition | null { + switch (op.op) { + case "rename_field": + return { ...def, key: op.to }; + case "set_field_localized": + return { ...def, localized: op.localized }; + case "remove_field": + return null; + case "change_field_type": + return retyped(def, op); + } +} + +/** Checks the request against the collection; throws when it can never run as written. */ +export function checkOperation(site: Site, op: FieldOperation): void { + const { defs, def } = currentDefinition(site, op); + if (op.op === "rename_field") { + if (!KEY.test(op.to)) + throw refuse("field_key_reserved", `${op.to} is not a field key (lower camelCase)`); + if (RESERVED_FIELD_KEYS.has(op.to)) + throw refuse("field_key_reserved", `${op.to} is reserved (a built-in entry property)`); + if (defs.some((d) => d.key === op.to)) + throw refuse("field_key_duplicate", `${op.to} is already a field of ${op.collection}`); + } + if (op.op === "change_field_type" && def.type === op.type && op.type !== "select") + throw new InvariantError("invalid_fields", `${def.key} already is a ${op.type} field`); +} + +/** Phase A's marker: the operation id and what readers fall back to. */ +export function markerOf(id: string, def: FieldDefinition, op: FieldOperation): FieldMigration { + switch (op.op) { + case "rename_field": + return { id, fromKey: def.key }; + case "change_field_type": + return { id, fromType: def.type }; + case "set_field_localized": + return { id, fromLocalized: def.localized }; + case "remove_field": + return { id }; + } +} + +/** The site with the collection's field list replaced. */ +export function withFields(site: Site, collection: string, fields: FieldDefinition[]): Site { + const settings = site.collections?.[collection] ?? {}; + const { fields: _old, ...rest } = settings; + const next = fields.length ? { ...rest, fields: fields as [FieldDefinition] } : rest; + return { ...site, collections: { ...site.collections, [collection]: next } }; +} + +/** Phase A: the target definition (or, for a removal, the field as it is) with the marker. */ +export function phaseASite(site: Site, op: FieldOperation, id: string): Site { + const { defs, def, index } = currentDefinition(site, op); + const marked = { ...(targetDefinition(def, op) ?? def), migration: markerOf(id, def, op) }; + const fields = defs.map((d, i) => (i === index ? (marked as FieldDefinition) : d)); + return withFields(site, op.collection, fields); +} + +/** The field carrying the operation's marker, if the site still has it. */ +export function markedField(site: Site, collection: string, id: string) { + const defs = site.collections?.[collection]?.fields ?? []; + const index = defs.findIndex((d) => d.migration?.id === id); + return index < 0 ? undefined : { defs, def: defs[index] as FieldDefinition, index }; +} + +/** Phase C: the marker dropped (a removed field goes entirely); undefined when it is gone. */ +export function phaseCSite(site: Site, collection: string, id: string): Site | undefined { + const found = markedField(site, collection, id); + if (!found) return undefined; + const { defs, def, index } = found; + const { migration, ...plain } = def; + const removing = migration && Object.keys(migration).length === 1; + const fields = removing + ? defs.filter((_, i) => i !== index) + : defs.map((d, i) => (i === index ? (plain as FieldDefinition) : d)); + return withFields(site, collection, fields); +} + +/** True when the site already shows the operation's end state (phase C ran). */ +export function isApplied(site: Site, op: FieldOperation, target: FieldDefinition | null): boolean { + const defs = site.collections?.[op.collection]?.fields ?? []; + if (defs.some((d) => d.migration)) return false; + const now = defs.find((d) => d.key === finalKey(op)); + if (op.op === "remove_field") return now === undefined; + if (!now || !target) return false; + if (op.op === "rename_field") return !defs.some((d) => d.key === op.from); + if (op.op === "set_field_localized") return now.localized === op.localized; + return ( + now.type === target.type && + (now.type !== "select" || + (target.type === "select" && (now.multiple === true) === (target.multiple === true))) + ); +} + +const IN_COLLECTION = + "site_id = ? AND kind = 'contentEntry' AND json_extract(body, '$.collection') = ?"; + +/** JSON types a value of `def` is stored as (phase C's mismatch test after a type change). */ +function jsonTypes(def: FieldDefinition): string[] { + if (def.type === "number") return ["integer", "real"]; + if (def.type === "boolean") return ["true", "false"]; + if (def.type === "datetime") return ["object"]; + if (def.type === "select" && def.multiple) return ["array"]; + return ["text"]; +} + +/** + * Entries of the collection the marker still has work for, as a SQL condition (one predicate + * per operation; keys match the FieldKey pattern, so they are safe in a JSON path). + */ +export function pendingCondition( + siteId: string, + collection: string, + def: FieldDefinition, +): Cond | undefined { + const m = def.migration; + if (!m) return undefined; + const path = `$.fields.${def.key}`; + const base = { sql: IN_COLLECTION, params: [siteId, collection] }; + if (m.fromKey !== undefined) + return { + sql: `${base.sql} AND json_type(body, ?) IS NOT NULL`, + params: [...base.params, `$.fields.${m.fromKey}`], + }; + if (m.fromType !== undefined) { + const types = jsonTypes(def); + return { + sql: `${base.sql} AND json_type(body, ?) IS NOT NULL AND json_type(body, ?) NOT IN (${types.map(() => "?").join(", ")})`, + params: [...base.params, path, path, ...types], + }; + } + if (m.fromLocalized !== undefined) { + if (def.localized) return undefined; + return { + sql: `${base.sql} AND json_type(body, ?) IS NOT NULL + AND json_extract(body, '$.translation.group') IS NOT NULL + AND json_extract(body, '$.translation.group') != id + AND json_extract(body, '$.translation.source.siteId') IS NULL`, + params: [...base.params, path], + }; + } + return { sql: `${base.sql} AND json_type(body, ?) IS NOT NULL`, params: [...base.params, path] }; +} + +/** The site still carries the operation's marker in the collection (phase B's guard). */ +export function markerGuard(siteId: string, collection: string, id: string): Cond { + return { + sql: `EXISTS (SELECT 1 FROM nodes s, json_each(s.body, '$.collections."${collection}".fields') f + WHERE s.id = ? AND s.kind = 'site' AND json_extract(f.value, '$.migration.id') = ?)`, + params: [siteId, id], + }; +} diff --git a/packages/core-ts/src/fields-operations/report.ts b/packages/core-ts/src/fields-operations/report.ts new file mode 100644 index 00000000..afb56c7a --- /dev/null +++ b/packages/core-ts/src/fields-operations/report.ts @@ -0,0 +1,97 @@ +/** + * RFC-0041 §6 step 1: the dry run's report. Entries touched, values moved, converted, cleared, + * removed, unchanged and unconvertible (in total and per locale), and up to five samples, the + * unconvertible ones first. The approval prompt and Studio's dialog are built from it; a type + * change with unconvertible values and no `onInvalid: "clear"` carries its refusal. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import type { Decide, Row } from "./rewrite.ts"; +import { isSameSiteTranslation } from "./rewrite.ts"; +import { + type FieldOperation, + type FieldOperationReport, + type FieldOperationSample, + type FieldValueCounts, + finalKey, + operationKey, +} from "./types.ts"; + +const SAMPLES = 5; +const SAMPLE_CHARS = 200; + +const zero = (): FieldValueCounts => ({ + touched: 0, + moved: 0, + converted: 0, + cleared: 0, + removed: 0, + unconvertible: 0, +}); + +/** Long strings shortened for a prompt (values are data, quoted by whoever shows them). */ +function short(value: unknown): unknown { + if (typeof value === "string" && value.length > SAMPLE_CHARS) + return `${value.slice(0, SAMPLE_CHARS)}…`; + return value; +} + +export interface ReportInput { + readonly op: FieldOperation; + readonly before: FieldDefinition; + readonly after: FieldDefinition | null; + readonly rows: readonly Row[]; + readonly decide: Decide; +} + +export function buildReport(input: ReportInput): FieldOperationReport { + const { op, rows, decide } = input; + const keys = [operationKey(op), finalKey(op)]; + const total = zero(); + const locales: Record = {}; + const samples: FieldOperationSample[] = []; + let unchanged = 0; + for (const row of rows) { + const e = row.entry; + const d = decide(row, isSameSiteTranslation(e)); + const change = d?.change; + if (!change) { + if (keys.some((k) => e.fields && Object.hasOwn(e.fields, k))) unchanged++; + continue; + } + const per = locales[e.locale] ?? zero(); + locales[e.locale] = per; + for (const c of [total, per]) { + c[change.kind]++; + if (d.fields || d.replace) c.touched++; + } + samples.push({ + entryId: e.id, + locale: e.locale, + kind: change.kind, + before: short(change.before), + after: short(change.after), + ...(change.reason ? { reason: change.reason } : {}), + }); + } + samples.sort((a, b) => Number(b.kind === "unconvertible") - Number(a.kind === "unconvertible")); + const clears = op.op === "change_field_type" && op.onInvalid === "clear"; + return { + op: op.op, + collection: op.collection, + key: operationKey(op), + entries: rows.length, + ...total, + unchanged, + locales, + samples: samples.slice(0, SAMPLES), + definition: { before: input.before, after: input.after }, + ...(total.unconvertible > 0 && !clears + ? { + refusal: { + code: "field_values_unconvertible" as const, + message: `${total.unconvertible} value(s) of ${operationKey(op)} cannot be converted to ${input.after?.type}; fix them first, or run with onInvalid: "clear" to drop them`, + }, + } + : {}), + }; +} diff --git a/packages/core-ts/src/fields-operations/revert.ts b/packages/core-ts/src/fields-operations/revert.ts new file mode 100644 index 00000000..0255405f --- /dev/null +++ b/packages/core-ts/src/fields-operations/revert.ts @@ -0,0 +1,198 @@ +/** + * RFC-0041 §6: `revertFieldOperation` restores the `before` snapshot's definition and the entries + * the operation touched, as `revertTagsToTerms` does, through the same phases: + * + * 1. the site CAS puts the earlier definition back with an inverse marker (a rename reads the new + * key as `fromKey`, a type change converts from the new type, …), so readers stay consistent; + * 2. entries exactly as the operation left them get their earlier body back (new revision); + * entries edited since are converted back through the inverse marker instead (`edited`); + * translations that were fresh stay fresh; + * 3. the site CAS drops the marker once nothing is pending, and a `revert …` snapshot closes it. + * + * Resumable: a call that ran out of budget answers `done: false`; calling again continues. + */ +import type { ContentEntry, FieldDefinition, FieldMigration, Site } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { convertContext, rewriteFields } from "../fields/migration.ts"; +import { sameJson } from "../fields/types.ts"; +import { markedField, markerGuard, pendingCondition, phaseCSite, withFields } from "./plan.ts"; +import { type Decision, loadEntries, type Row, rewritePass } from "./rewrite.ts"; +import { beforeSiteOf } from "./run.ts"; +import { + labelOf, + nothingPending, + readOperation, + refuseRunning, + requireSite, + writeSite, +} from "./state.ts"; +import { + FIELD_MIGRATION, + type FieldOperation, + type FieldOperationOptions, + type FieldRevertResult, + finalKey, + META, + type OperationContext, + operationContext, + operationKey, +} from "./types.ts"; + +/** The marker that reads the operation's end state as the earlier one's source. */ +function inverseMarker( + id: string, + op: FieldOperation, + now: FieldDefinition | undefined, +): FieldMigration | undefined { + switch (op.op) { + case "rename_field": + return { id, fromKey: op.to }; + case "change_field_type": + return now ? { id, fromType: now.type } : undefined; + case "set_field_localized": + return { id, fromLocalized: op.localized }; + case "remove_field": + return undefined; + } +} + +/** Step 1: the earlier definition back in place, with the inverse marker. */ +function restoredSite(site: Site, op: FieldOperation, beforeSite: Site, id: string): Site { + const was = beforeSite.collections?.[op.collection]?.fields ?? []; + const index = was.findIndex((d) => d.key === operationKey(op)); + const earlier = was[index] as FieldDefinition; + const defs = [...(site.collections?.[op.collection]?.fields ?? [])]; + const at = defs.findIndex((d) => d.key === finalKey(op)); + const marker = inverseMarker(id, op, defs[at]); + const def = (marker ? { ...earlier, migration: marker } : earlier) as FieldDefinition; + if (at >= 0) defs[at] = def; + else defs.splice(Math.min(index, defs.length), 0, def); + return withFields(site, op.collection, defs); +} + +/** The entry without what an operation writes: values, the source it points at, stamps. */ +function rest(entry: ContentEntry) { + const { fields: _f, rev: _r, updatedAt: _u, translation, ...others } = entry; + const { source: _s, ...t } = translation ?? {}; + return { ...others, translation: t }; +} + +interface Snapshots { + readonly before: Map; + readonly after: Map; +} + +/** The earlier bodies of entries the operation alone changed and nobody edited since. */ +async function restorable( + ctx: OperationContext, + rows: readonly Row[], + snaps: Snapshots, +): Promise> { + const out = new Map(); + for (const row of rows) { + const id = row.entry.id; + const [was, then] = [snaps.before.get(id), snaps.after.get(id)]; + if (!was || !then || was === then || row.hash !== then) continue; + const earlier = await ctx.store.readSnapshotNode({ + id, + kind: "contentEntry", + rev: 0, + hash: was, + }); + if (sameJson(rest(earlier), rest(row.entry))) out.set(id, earlier); + } + return out; +} + +export async function revertFieldOperation( + store: OperationContext["store"], + siteId: string, + operationId: string, + options: FieldOperationOptions, +): Promise { + const ctx = operationContext(store, options); + const record = await readOperation(ctx, siteId, operationId); + const { op, before, after } = record; + if (!after) + throw new InvariantError( + "field_operation_incomplete", + `field operation ${operationId} did not finish; continue it, then revert`, + { operationId }, + ); + if (record.reverted) + throw new InvariantError( + "field_operation_incomplete", + `field operation ${operationId} was already reverted (${record.reverted.id})`, + { operationId }, + ); + const beforeSite = await beforeSiteOf(ctx, before); + let site = await requireSite(ctx, siteId); + const removal = op.op === "remove_field"; + const restoredDef = (site.collections?.[op.collection]?.fields ?? []).some((d) => + removal ? d.key === op.key : d.migration?.id === operationId, + ); + if (!restoredDef) { + const running = (site.collections?.[op.collection]?.fields ?? []).find((d) => d.migration); + if (running?.migration) refuseRunning(op.collection, running.migration.id); + await writeSite(ctx, siteId, (s) => restoredSite(s, op, beforeSite, operationId)); + } + const snaps: Snapshots = { + before: new Map(before.nodes.map((n) => [n.id, n.hash])), + after: new Map(after.nodes.map((n) => [n.id, n.hash])), + }; + const out: FieldRevertResult = { operationId, done: false, restored: 0, edited: [], skipped: [] }; + for (let pass = 0; pass < 3; pass++) { + site = await requireSite(ctx, siteId); + const marked = markedField(site, op.collection, operationId); + const rows = await loadEntries(ctx, siteId, op.collection); + const earlier = await restorable(ctx, rows, snaps); + const convert = convertContext(site); + const decide = (row: Row, translation: boolean): Decision | undefined => { + const body = earlier.get(row.entry.id); + if (body) return { replace: body }; + if (!marked) return undefined; + const r = rewriteFields({ + def: marked.def, + values: row.entry.fields, + convert, + translation, + pristine: false, + }); + return r.values ? { fields: r.values, change: r.change } : undefined; + }; + const r = await rewritePass({ + ctx, + site, + rows, + defs: site.collections?.[op.collection]?.fields ?? [], + decide, + guard: marked ? markerGuard(siteId, op.collection, operationId) : undefined, + freshAlso: (sourceId) => (earlier.has(sourceId) ? snaps.before.get(sourceId) : undefined), + done: out.restored + out.edited.length, + }); + for (const w of r.written) + if (w.decision?.replace) out.restored++; + else if (w.decision?.fields) out.edited.push(w.id); + out.skipped = r.skipped; + if (r.exhausted) return out; + if (r.skipped.length === 0) break; + } + const marked = markedField(site, op.collection, operationId); + if (marked) { + const pending = pendingCondition(siteId, op.collection, marked.def); + const wrote = await writeSite( + ctx, + siteId, + (s) => phaseCSite(s, op.collection, operationId), + nothingPending(pending), + ); + if (wrote === "guarded") return out; + } + const snapshot = await store.createSnapshot(siteId, { + actor: options.actor, + reason: "manual", + label: labelOf("revert ", op), + metadata: { [META.kind]: FIELD_MIGRATION, [META.id]: operationId, [META.phase]: "reverted" }, + }); + return { ...out, done: true, skipped: [], snapshotId: snapshot.id }; +} diff --git a/packages/core-ts/src/fields-operations/rewrite.ts b/packages/core-ts/src/fields-operations/rewrite.ts new file mode 100644 index 00000000..50ef4922 --- /dev/null +++ b/packages/core-ts/src/fields-operations/rewrite.ts @@ -0,0 +1,184 @@ +/** + * RFC-0041 §6 phase B (and the revert's): entries rewritten in guarded batches. Each entry is its + * own CAS on the revision read, guarded on the site still carrying the operation's marker; an + * entry changed meanwhile is skipped and planned again from what is stored on the next pass. A + * translation that was fresh (its `translation.source.hash` was its source's hash) is re-pointed + * at the source's new hash in the same pass, guarded on that new hash (rule 4), so an operation + * never makes translations look stale. `asset_refs` rows under `fields.*` follow each entry. + */ +import { + assertValid, + type ContentEntry, + type FieldDefinition, + type Site, +} from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import type { ValueChange } from "../fields/migration.ts"; +import { nodeHash } from "../hash.ts"; +import { fieldAssetRefs } from "../media/field-refs.ts"; +import { and, type Cond, casUpdateStatement, hashIs } from "../node-sql.ts"; +import { fieldRefStatements } from "../store/entry-fields.ts"; +import { forwardedSource } from "../store/source-forward.ts"; +import { FIELD_OPERATION_BATCH, type OperationContext } from "./types.ts"; + +export interface Row { + readonly entry: ContentEntry; + readonly hash: string; +} + +/** What happens to one entry: new `fields`, or a whole earlier body (a revert's restore). */ +export interface Decision { + readonly fields?: Record | undefined; + readonly replace?: ContentEntry | undefined; + readonly change?: ValueChange | undefined; +} + +export type Decide = (row: Row, translation: boolean) => Decision | undefined; + +/** Every entry of a collection, oldest first. */ +export async function loadEntries( + ctx: OperationContext, + siteId: string, + collection: string, +): Promise { + const rows = await ctx.store.sql.all<{ body: string; hash: string }>( + `SELECT body, hash FROM nodes WHERE site_id = ? AND kind = 'contentEntry' + AND json_extract(body, '$.collection') = ? ORDER BY created_at, id`, + [siteId, collection], + ); + return rows.map((r) => ({ entry: JSON.parse(r.body) as ContentEntry, hash: r.hash })); +} + +/** A translation whose original is in the same site (RFC-0041 §5). */ +export function isSameSiteTranslation(entry: ContentEntry): boolean { + const t = entry.translation; + return !!t && t.group !== entry.id && t.source?.siteId === undefined; +} + +interface Planned { + readonly stored: Row; + readonly next: ContentEntry; + readonly hash: string; + readonly fieldsChanged: boolean; + readonly anchor: Cond | undefined; +} + +export interface PassInput { + readonly ctx: OperationContext; + readonly site: Site; + readonly rows: readonly Row[]; + /** The collection's definitions as stored now (for `asset_refs`). */ + readonly defs: readonly FieldDefinition[]; + readonly decide: Decide; + /** Every statement's guard: the site still carries the marker (absent: none). */ + readonly guard: Cond | undefined; + /** A recorded source hash that counts as fresh besides the source's current one. */ + readonly freshAlso?: ((sourceId: string) => string | undefined) | undefined; + /** Entries this call already rewrote (the budget counts across passes). */ + readonly done: number; +} + +export interface PassResult { + rewritten: number; + skipped: { entryId: string; reason: string }[]; + /** Ids written by this pass with their decision (revert reports restored and edited ones). */ + written: { id: string; decision: Decision | undefined }[]; + /** The budget ran out before every planned entry was written. */ + exhausted: boolean; +} + +/** Sources before the translations made from them (chains are short; capped at 8). */ +function depthOf(row: Row, byId: ReadonlyMap): number { + let depth = 0; + let at: Row | undefined = row; + while (at && depth < 8) { + const src = at.entry.translation?.source; + if (!src || src.siteId !== undefined) break; + at = byId.get(src.id); + if (at) depth++; + } + return depth; +} + +/** Plans every entry that changes: its decision and, for fresh translations, the new source. */ +async function planPass(input: PassInput, ts: string) { + const byId = new Map(input.rows.map((r) => [r.entry.id, r])); + const order = [...input.rows].sort((a, b) => depthOf(a, byId) - depthOf(b, byId)); + const planned = new Map(); + const decisions = new Map(); + for (const row of order) { + const d = input.decide(row, isSameSiteTranslation(row.entry)); + decisions.set(row.entry.id, d); + let body: ContentEntry | undefined = + d?.replace ?? (d?.fields ? { ...row.entry, fields: d.fields } : undefined); + let anchor: Cond | undefined; + const src = (body ?? row.entry).translation?.source; + const from = src && src.siteId === undefined ? planned.get(src.id) : undefined; + if (src && from) { + const fresh = src.hash === from.stored.hash || src.hash === input.freshAlso?.(src.id); + if (fresh) { + body = forwardedSource(body ?? row.entry, { rev: from.next.rev, hash: from.hash }, ts); + body = { ...body, rev: row.entry.rev }; + anchor = hashIs(from.next.id, from.hash); + } + } + if (!body) continue; + const next: ContentEntry = { ...body, rev: row.entry.rev + 1, updatedAt: ts }; + assertValid("content-entry", next); + planned.set(row.entry.id, { + stored: row, + next, + hash: await nodeHash(next), + fieldsChanged: d?.fields !== undefined || d?.replace !== undefined, + anchor, + }); + } + return { planned: [...planned.values()], decisions }; +} + +/** The statements of one entry: its CAS, then its `fields.*` asset rows when values moved. */ +async function statementsOf(input: PassInput, p: Planned): Promise { + const cas = await casUpdateStatement( + "contentEntry", + p.next, + p.stored.entry.rev, + and(input.guard, p.anchor), + ); + if (!p.fieldsChanged) return [cas]; + const refs = fieldAssetRefs(input.defs, p.next.fields); + return [cas, ...fieldRefStatements(input.site.id, p.next.id, refs, p.hash)]; +} + +const outOfBudget = (ctx: OperationContext, done: number) => + done >= ctx.budget.entries || Date.now() - ctx.started >= ctx.budget.ms; + +/** One pass over the rows: plan, then write in batches until done or out of budget. */ +export async function rewritePass(input: PassInput): Promise { + const { ctx } = input; + const { planned, decisions } = await planPass(input, ctx.stamp()); + const out: PassResult = { rewritten: 0, skipped: [], written: [], exhausted: false }; + let i = 0; + while (i < planned.length) { + if (outOfBudget(ctx, input.done + out.rewritten)) { + out.exhausted = true; + break; + } + const statements: SqlStatement[] = []; + const heads: { index: number; p: Planned }[] = []; + const room = ctx.budget.entries - input.done - out.rewritten; + while (i < planned.length && statements.length < FIELD_OPERATION_BATCH && heads.length < room) { + const p = planned[i++] as Planned; + heads.push({ index: statements.length, p }); + statements.push(...(await statementsOf(input, p))); + } + const results = await ctx.store.sql.batch(statements); + for (const { index, p } of heads) { + const id = p.next.id; + if (results[index]?.changes === 1) { + out.rewritten++; + out.written.push({ id, decision: decisions.get(id) }); + } else out.skipped.push({ entryId: id, reason: "changed during the operation; retried" }); + } + } + return out; +} diff --git a/packages/core-ts/src/fields-operations/run.ts b/packages/core-ts/src/fields-operations/run.ts new file mode 100644 index 00000000..6641c311 --- /dev/null +++ b/packages/core-ts/src/fields-operations/run.ts @@ -0,0 +1,295 @@ +/** + * RFC-0041 §6: running a field operation. Dry run first (always); then the `before` snapshot, + * phase A (the site CAS writes the new definition with its `migration` marker), phase B (entries + * in guarded batches, resumable), phase C (the site CAS drops the marker, guarded by `NOT EXISTS` + * over what is still pending) and the `after` snapshot. Only the draft graph changes: the published + * snapshot keeps the old definitions with the old values until the next publish. + */ +import type { FieldDefinition, Site, Snapshot } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { convertContext, rewriteFields } from "../fields/migration.ts"; +import { + checkOperation, + currentDefinition, + isApplied, + markedField, + markerGuard, + pendingCondition, + phaseASite, + phaseCSite, + runningMarker, + targetDefinition, +} from "./plan.ts"; +import { buildReport } from "./report.ts"; +import { type Decide, loadEntries, rewritePass } from "./rewrite.ts"; +import { + labelOf, + nothingPending, + type OperationRecord, + readOperation, + refuseRunning, + requireSite, + writeSite, +} from "./state.ts"; +import { + FIELD_MIGRATION, + type FieldOperation, + type FieldOperationOptions, + type FieldOperationReport, + type FieldOperationResult, + type FieldOperationStatus, + META, + type OperationContext, + operationContext, +} from "./types.ts"; + +/** Passes per call while concurrent edits keep entries skipped. */ +const PASSES = 3; +const DRY_RUN_ID = "snap_fielddryrun"; + +/** How entries change under `def`'s marker; `pristine` per entry from the `before` snapshot. */ +function decider( + site: Site, + def: FieldDefinition, + op: FieldOperation, + pristine: (id: string, hash: string) => boolean, +): Decide { + const convert = convertContext(site); + const onInvalid = op.op === "change_field_type" ? op.onInvalid : undefined; + return (row, translation) => { + const r = rewriteFields({ + def, + values: row.entry.fields, + convert, + translation, + pristine: pristine(row.entry.id, row.hash), + onInvalid, + }); + return r.values || r.change ? { fields: r.values, change: r.change } : undefined; + }; +} + +/** The dry run of `op` on the site as it is (every entry counts as untouched). */ +async function dryRun( + ctx: OperationContext, + site: Site, + op: FieldOperation, +): Promise { + const { def } = currentDefinition(site, op); + const marked = markedField(phaseASite(site, op, DRY_RUN_ID), op.collection, DRY_RUN_ID); + const rows = await loadEntries(ctx, site.id, op.collection); + return buildReport({ + op, + before: def, + after: targetDefinition(def, op), + rows, + decide: decider(site, marked?.def as FieldDefinition, op, () => true), + }); +} + +function result( + record: Pick & { after?: Snapshot | undefined }, + report: FieldOperationReport, + extra: Partial, +): FieldOperationResult { + return { + operationId: record.before.id, + dryRun: false, + done: record.after !== undefined, + resumed: false, + report, + rewritten: 0, + skipped: [], + beforeSnapshotId: record.before.id, + ...(record.after ? { afterSnapshotId: record.after.id } : {}), + ...extra, + }; +} + +/** + * Runs `op` (or only reports it with `dryRun`). Refuses while another operation rewrites the + * collection (`operation_in_progress`) and, for a type change, values the table cannot convert + * without `onInvalid: "clear"` (`field_values_unconvertible`, the report in `details.report`). + * Answers `done: false` when the budget ran out; `continueFieldOperation` resumes. + */ +export async function runFieldOperation( + store: OperationContext["store"], + siteId: string, + op: FieldOperation, + options: FieldOperationOptions & { dryRun?: boolean }, +): Promise { + const ctx = operationContext(store, options); + const site = await requireSite(ctx, siteId); + checkOperation(site, op); + const running = runningMarker(site.collections?.[op.collection]?.fields ?? []); + if (running) refuseRunning(op.collection, running.id); + const report = await dryRun(ctx, site, op); + if (options.dryRun) + return { + operationId: "", + dryRun: true, + done: false, + resumed: false, + report, + rewritten: 0, + skipped: [], + }; + if (report.refusal) + throw new InvariantError("field_values_unconvertible", report.refusal.message, { report }); + const before = await store.createSnapshot(siteId, { + actor: options.actor, + reason: "manual", + label: labelOf("before ", op), + metadata: { + [META.kind]: FIELD_MIGRATION, + [META.phase]: "before", + [META.op]: JSON.stringify(op), + }, + }); + return proceed(ctx, { before, op, after: undefined, reverted: undefined }, report, options); +} + +/** Resumes an interrupted or budget-limited operation from its marker. */ +export async function continueFieldOperation( + store: OperationContext["store"], + siteId: string, + operationId: string, + options: FieldOperationOptions, +): Promise { + const ctx = operationContext(store, options); + const record = await readOperation(ctx, siteId, operationId); + const site = await requireSite(ctx, siteId); + const report = await remaining(ctx, site, record); + if (record.after || record.reverted) return result(record, report, { resumed: true }); + return { ...(await proceed(ctx, record, report, options)), resumed: true }; +} + +/** The report of what is left: through the marker when phase A ran, else the dry run. */ +async function remaining( + ctx: OperationContext, + site: Site, + record: OperationRecord, +): Promise { + const { op, before } = record; + const marked = markedField(site, op.collection, before.id); + const beforeSite = await beforeSiteOf(ctx, before); + const { def } = currentDefinition(beforeSite, op); + const closed = record.after !== undefined || record.reverted !== undefined; + if (!marked && !closed && !isApplied(site, op, targetDefinition(def, op))) + return dryRun(ctx, site, op); + const hashes = new Map(before.nodes.map((n) => [n.id, n.hash])); + return buildReport({ + op, + before: def, + after: targetDefinition(def, op), + rows: marked ? await loadEntries(ctx, site.id, op.collection) : [], + decide: marked + ? decider(site, marked.def, op, (id, hash) => hashes.get(id) === hash) + : () => undefined, + }); +} + +/** The site node as the `before` snapshot captured it. */ +export async function beforeSiteOf(ctx: OperationContext, before: Snapshot): Promise { + const ref = before.nodes.find((n) => n.kind === "site"); + if (!ref) throw new InvariantError("field_operation_incomplete", "the snapshot has no site"); + return ctx.store.readSnapshotNode(ref); +} + +/** Phases A, B and C from wherever the operation stands. */ +async function proceed( + ctx: OperationContext, + record: OperationRecord, + report: FieldOperationReport, + options: FieldOperationOptions, +): Promise { + const { op, before } = record; + const id = before.id; + const siteId = before.siteId; + let site = await requireSite(ctx, siteId); + if (!markedField(site, op.collection, id)) { + const { def } = currentDefinition(await beforeSiteOf(ctx, before), op); + if (!isApplied(site, op, targetDefinition(def, op))) { + const running = runningMarker(site.collections?.[op.collection]?.fields ?? []); + if (running) refuseRunning(op.collection, running.id); + await writeSite(ctx, siteId, (s) => phaseASite(s, op, id)); + site = await requireSite(ctx, siteId); + } + } + const hashes = new Map(before.nodes.map((n) => [n.id, n.hash])); + let rewritten = 0; + let skipped: FieldOperationResult["skipped"] = []; + for (let pass = 0; pass < PASSES; pass++) { + site = await requireSite(ctx, siteId); + const marked = markedField(site, op.collection, id); + if (!marked) break; + const r = await rewritePass({ + ctx, + site, + rows: await loadEntries(ctx, siteId, op.collection), + defs: marked.defs, + decide: decider(site, marked.def, op, (eid, hash) => hashes.get(eid) === hash), + guard: markerGuard(siteId, op.collection, id), + done: rewritten, + }); + rewritten += r.rewritten; + skipped = r.skipped; + if (r.exhausted) return result(record, report, { rewritten, skipped, done: false }); + if (skipped.length === 0) break; + } + const marked = markedField(site, op.collection, id); + if (marked) { + const pending = pendingCondition(siteId, op.collection, marked.def); + const wrote = await writeSite( + ctx, + siteId, + (s) => phaseCSite(s, op.collection, id), + nothingPending(pending), + ); + if (wrote === "guarded") return result(record, report, { rewritten, skipped, done: false }); + } + const after = await ctx.store.createSnapshot(siteId, { + actor: options.actor, + reason: "manual", + label: labelOf("", op), + metadata: { [META.kind]: FIELD_MIGRATION, [META.id]: id, [META.phase]: "after" }, + }); + return result({ before, after }, report, { rewritten, skipped: [], done: true }); +} + +/** Where an operation stands, with the entries still to rewrite or restore. */ +export async function fieldOperationStatus( + store: OperationContext["store"], + siteId: string, + operationId: string, +): Promise { + const ctx = operationContext(store, {}); + const record = await readOperation(ctx, siteId, operationId); + const site = await requireSite(ctx, siteId); + const marked = markedField(site, record.op.collection, operationId); + let pending = 0; + const cond = marked ? pendingCondition(siteId, record.op.collection, marked.def) : undefined; + if (cond) { + const row = await store.sql.get<{ n: number }>( + `SELECT count(*) AS n FROM nodes WHERE ${cond.sql}`, + cond.params, + ); + pending = Number(row?.n ?? 0); + } + const state = record.reverted + ? "reverted" + : record.after + ? marked + ? "reverting" + : "done" + : "running"; + return { + operationId, + operation: record.op, + state, + pending, + beforeSnapshotId: record.before.id, + ...(record.after ? { afterSnapshotId: record.after.id } : {}), + ...(record.reverted ? { revertSnapshotId: record.reverted.id } : {}), + }; +} diff --git a/packages/core-ts/src/fields-operations/state.ts b/packages/core-ts/src/fields-operations/state.ts new file mode 100644 index 00000000..6b222759 --- /dev/null +++ b/packages/core-ts/src/fields-operations/state.ts @@ -0,0 +1,105 @@ +/** + * RFC-0041 §6: an operation's durable state. The `before` snapshot (its id is the operation id) + * keeps the request in its metadata; the site's `migration` marker says phase A ran and C has + * not; the `after` and `reverted` snapshots close it. Site writes of phases A and C are the site + * CAS itself, not `updateSite`: the plain-edit guard refuses markers by design, so only these + * writes may set or drop one. They still pass the precompiled validator and the definition rules. + */ +import { assertValid, type Site, type Snapshot } from "@cloudhal/core-contracts"; +import { ConflictError, InvariantError, NotFoundError } from "../errors.ts"; +import { checkFieldDefinitions, refuseFieldIssues } from "../fields/index.ts"; +import { type Cond, casUpdateStatement } from "../node-sql.ts"; +import { + FIELD_MIGRATION, + type FieldOperation, + META, + type OperationContext, + operationKey, +} from "./types.ts"; + +export interface OperationRecord { + readonly before: Snapshot; + readonly op: FieldOperation; + readonly after: Snapshot | undefined; + readonly reverted: Snapshot | undefined; +} + +/** The snapshot label of a phase: `before rename_field events.speaker`. */ +export const labelOf = (prefix: string, op: FieldOperation) => + `${prefix}${op.op} ${op.collection}.${operationKey(op)}`.slice(0, 200); + +/** The operation `id` of `siteId`, or NotFoundError. */ +export async function readOperation( + ctx: OperationContext, + siteId: string, + id: string, +): Promise { + const before = await ctx.store.getSnapshot(id); + const meta = before?.metadata; + if ( + !before || + before.siteId !== siteId || + meta?.[META.kind] !== FIELD_MIGRATION || + meta[META.phase] !== "before" || + !meta[META.op] + ) + throw new NotFoundError("field operation", id); + const later = (await ctx.store.listSnapshots(siteId, 1000)).filter( + (s) => s.metadata?.[META.kind] === FIELD_MIGRATION && s.metadata[META.id] === id, + ); + return { + before, + op: JSON.parse(meta[META.op] as string) as FieldOperation, + after: later.find((s) => s.metadata?.[META.phase] === "after"), + reverted: later.find((s) => s.metadata?.[META.phase] === "reverted"), + }; +} + +export async function requireSite(ctx: OperationContext, siteId: string): Promise { + const site = await ctx.store.getSite(siteId); + if (!site) throw new NotFoundError("site", siteId); + return site; +} + +export type SiteWrite = "written" | "unchanged" | "guarded"; + +/** + * Writes `transform(site)` with the site CAS (plus `guard`), re-reading on a concurrent site + * write. `unchanged` when the transform has nothing to do, `guarded` when `guard` failed. + */ +export async function writeSite( + ctx: OperationContext, + siteId: string, + transform: (site: Site) => Site | undefined, + guard?: Cond, +): Promise { + let rev: number | undefined; + for (let attempt = 0; attempt < 5; attempt++) { + const site = await requireSite(ctx, siteId); + const changed = transform(site); + if (!changed) return "unchanged"; + const next: Site = { ...changed, rev: site.rev + 1, updatedAt: ctx.stamp() }; + assertValid("site", next); + refuseFieldIssues(checkFieldDefinitions(next)); + const s = await casUpdateStatement("site", next, site.rev, guard); + const r = await ctx.store.sql.run(s.sql, s.params ?? []); + if (r.changes === 1) return "written"; + rev = (await ctx.store.getSite(siteId))?.rev; + if (rev === site.rev) return "guarded"; + } + throw new ConflictError("site", siteId, rev ?? 0, rev); +} + +/** `NOT EXISTS` over a pending-entries condition (phase C's guard). */ +export function nothingPending(pending: Cond | undefined): Cond | undefined { + if (!pending) return undefined; + return { sql: `NOT EXISTS (SELECT 1 FROM nodes WHERE ${pending.sql})`, params: pending.params }; +} + +export function refuseRunning(collection: string, id: string): never { + throw new InvariantError( + "operation_in_progress", + `field operation ${id} is still rewriting ${collection}; continue or revert it first`, + { operationId: id, collection }, + ); +} diff --git a/packages/core-ts/src/fields-operations/types.ts b/packages/core-ts/src/fields-operations/types.ts new file mode 100644 index 00000000..b3c555df --- /dev/null +++ b/packages/core-ts/src/fields-operations/types.ts @@ -0,0 +1,175 @@ +/** + * RFC-0041 §6: the four field operations, their reports and results. Shared by the runner, the + * API routes and the CLI. + */ +import type { Actor, FieldDefinition, FieldOption } from "@cloudhal/core-contracts"; +import type { FieldType } from "../fields/types.ts"; +import type { SiteGraphStore } from "../store.ts"; + +/** `metadata["cloudhal.migration"]` of an operation's snapshots. */ +export const FIELD_MIGRATION = "0041-field"; +export const META = { + kind: "cloudhal.migration", + id: "cloudhal.migration.id", + phase: "cloudhal.migration.phase", + op: "cloudhal.migration.op", +} as const; +/** Entries rewritten per batch (as the tags migration). */ +export const FIELD_OPERATION_BATCH = 200; +/** What one request does before it answers `done: false` (RFC-0041 §6, "Runs and budgets"). */ +export const FIELD_OPERATION_BUDGET = { entries: 2000, ms: 20_000 } as const; + +export type FieldOperationName = + | "rename_field" + | "change_field_type" + | "set_field_localized" + | "remove_field"; + +export const FIELD_OPERATIONS: readonly FieldOperationName[] = [ + "rename_field", + "change_field_type", + "set_field_localized", + "remove_field", +]; + +/** One operation as requested: what an agent tool, the API and the CLI send. */ +export type FieldOperation = + | { op: "rename_field"; collection: string; from: string; to: string } + | { + op: "change_field_type"; + collection: string; + key: string; + type: FieldType; + /** The options of a new select (default: the field's own when it already is one). */ + options?: FieldOption[] | undefined; + multiple?: boolean | undefined; + /** Drop values the table cannot convert; without it such values refuse the operation. */ + onInvalid?: "clear" | undefined; + } + | { op: "set_field_localized"; collection: string; key: string; localized: boolean } + | { op: "remove_field"; collection: string; key: string }; + +/** Counts of what happens to values (per locale too). */ +export interface FieldValueCounts { + /** Entries whose `fields` change. */ + touched: number; + /** Values moved to the new key. */ + moved: number; + /** Values converted to the new type. */ + converted: number; + /** Values dropped: unconvertible with `onInvalid: "clear"`, or translations' copies. */ + cleared: number; + /** Values deleted by `remove_field`. */ + removed: number; + /** Values the table cannot convert (refuse the operation unless `onInvalid: "clear"`). */ + unconvertible: number; +} + +export interface FieldOperationSample { + entryId: string; + locale: string; + kind: "moved" | "converted" | "cleared" | "removed" | "unconvertible"; + before: unknown; + after: unknown; + reason?: string; +} + +/** The dry run: what the approval prompt and Studio's dialog are built from. */ +export interface FieldOperationReport extends FieldValueCounts { + op: FieldOperationName; + collection: string; + /** The field as it is (for a rename, its current key). */ + key: string; + /** Entries of the collection read. */ + entries: number; + /** Entries holding a value that stays as it is. */ + unchanged: number; + /** The same counts per locale. */ + locales: Record; + /** Up to 5 values before and after (unconvertible ones first). */ + samples: FieldOperationSample[]; + definition: { before: FieldDefinition; after: FieldDefinition | null }; + /** Why running it would be refused (values the table cannot convert, without `clear`). */ + refusal?: { code: "field_values_unconvertible"; message: string } | undefined; +} + +export interface FieldOperationResult { + /** The `before` snapshot's id; empty for a dry run of an operation not started. */ + operationId: string; + dryRun: boolean; + /** Every phase ran: the marker is gone and the `after` snapshot exists. */ + done: boolean; + /** An interrupted or budget-limited run was continued. */ + resumed: boolean; + report: FieldOperationReport; + /** Entries rewritten by this call. */ + rewritten: number; + /** Entries changed meanwhile, retried by the next call. */ + skipped: { entryId: string; reason: string }[]; + beforeSnapshotId?: string; + afterSnapshotId?: string; +} + +export type FieldOperationState = "running" | "done" | "reverting" | "reverted"; + +export interface FieldOperationStatus { + operationId: string; + operation: FieldOperation; + state: FieldOperationState; + /** Entries still to rewrite (running) or to restore (reverting). */ + pending: number; + beforeSnapshotId: string; + afterSnapshotId?: string; + revertSnapshotId?: string; +} + +export interface FieldRevertResult { + operationId: string; + done: boolean; + /** Entries given their earlier values back exactly. */ + restored: number; + /** Entries edited since the operation: converted back through the marker instead. */ + edited: string[]; + skipped: { entryId: string; reason: string }[]; + snapshotId?: string; +} + +/** What every call takes. */ +export interface FieldOperationOptions { + actor: Actor; + /** The clock of `updatedAt` stamps (default: now). */ + now?: (() => Date) | undefined; + /** Stop after this much and answer `done: false` (default `FIELD_OPERATION_BUDGET`). */ + budget?: { entries?: number; ms?: number } | undefined; +} + +/** The store plus the clock. */ +export interface OperationContext { + readonly store: SiteGraphStore; + readonly stamp: () => string; + readonly started: number; + readonly budget: { entries: number; ms: number }; +} + +export function operationContext( + store: SiteGraphStore, + options: Pick, +): OperationContext { + const now = options.now ?? (() => new Date()); + return { + store, + stamp: () => now().toISOString(), + started: Date.now(), + budget: { + entries: options.budget?.entries ?? FIELD_OPERATION_BUDGET.entries, + ms: options.budget?.ms ?? FIELD_OPERATION_BUDGET.ms, + }, + }; +} + +/** The field key the operation names (a rename's current key). */ +export const operationKey = (op: FieldOperation): string => + op.op === "rename_field" ? op.from : op.key; + +/** The field key once the operation is done. */ +export const finalKey = (op: FieldOperation): string => (op.op === "rename_field" ? op.to : op.key); diff --git a/packages/core-ts/src/fields/conversions.ts b/packages/core-ts/src/fields/conversions.ts new file mode 100644 index 00000000..0da3748e --- /dev/null +++ b/packages/core-ts/src/fields/conversions.ts @@ -0,0 +1,231 @@ +/** + * RFC-0041 §6: the closed conversion table of `change_field_type`, shared by the operation's dry + * run and rewrite and by `resolveFields` (which converts values still in `fromType` while an + * operation runs). Pure, Worker-safe and Temporal-free: zones go through `Intl` only. + * + * | from → to | conversion | + * |---|---| + * | text → richText | Markdown punctuation escaped, so the words read the same | + * | richText → text | plain text (markup, links' targets and directives dropped) | + * | text → number | a decimal number, nothing else | + * | number → text | its decimal form | + * | text → select | an option's value, or its label in the site's default language | + * | select → text | the option value(s), comma separated | + * | text ↔ url | the same string, when it is a link the url type accepts | + * | date → datetime | 00:00 of the day in the site's zone | + * | datetime → date | the day in the value's own zone | + * | select ↔ select | single ↔ multiple (one value ↔ a list of one; an empty list is no value) | + * + * Every result must also fit the new definition (`valueIssues`); anything else is unconvertible. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import type { FieldType } from "./types.ts"; +import { valueIssues } from "./value-checks.ts"; + +/** What a conversion needs from the site. */ +export interface ConvertContext { + /** The site's zone (`date → datetime`); `UTC` when the site has none. */ + readonly timeZone: string; + /** The site's default language (`text → select` labels). */ + readonly defaultLocale: string; +} + +/** `value: undefined` means "no value" (the key is removed). */ +export type Conversion = { ok: true; value: unknown } | { ok: false; reason: string }; + +/** The table, from → the types it converts to (besides itself). */ +export const FIELD_CONVERSIONS: Readonly>> = { + text: ["richText", "number", "select", "url"], + richText: ["text"], + number: ["text"], + select: ["text", "select"], + url: ["text"], + date: ["datetime"], + datetime: ["date"], +}; + +/** True when the table converts `from` values to `to` (a type always converts to itself). */ +export function canConvert(from: FieldType, to: FieldType): boolean { + return from === to || FIELD_CONVERSIONS[from]?.includes(to) === true; +} + +type Step = (value: unknown, to: FieldDefinition, ctx: ConvertContext) => Conversion; + +const fail = (reason: string): Conversion => ({ ok: false, reason }); +const ok = (value: unknown): Conversion => ({ ok: true, value }); + +/** CommonMark escapes for the characters that would turn plain words into markup. */ +export function escapeMarkdown(text: string): string { + return text + .replace(/[\\`*_[\]<>|~&]/g, "\\$&") + .replace(/^(\s*)([#>+=-])/gm, "$1\\$2") + .replace(/^(\s*\d+)([.)])/gm, "$1\\$2") + .replace(/:(?=:)/g, "\\:"); +} + +/** The words of a markdown string: no markup, link text without targets, no directives. */ +export function markdownToPlain(markdown: string, multiline: boolean): string { + const lines: string[] = []; + for (const raw of markdown.split(/\r?\n/)) { + const line = raw.trim(); + if (line.startsWith("```") || line.startsWith("~~~") || line.startsWith("::")) continue; + const words = line + .replace(/^#{1,6}\s+/, "") + .replace(/^>\s?/, "") + .replace(/^(?:[-*+]|\d+[.)])\s+/, "") + .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/(\*\*|__|~~|\*|_|`)/g, "") + .replace(/\\([!-/:-@[-`{-~])/g, "$1") + .trim(); + if (words) lines.push(words); + } + return lines.join(multiline ? "\n" : " "); +} + +const formats = new Map(); +function partsIn(zone: string, epochMs: number): Record { + let f = formats.get(zone); + if (!f) { + f = new Intl.DateTimeFormat("en-US", { + timeZone: zone, + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + formats.set(zone, f); + } + const out: Record = {}; + for (const p of f.formatToParts(new Date(epochMs))) + if (p.type !== "literal") out[p.type] = Number(p.value); + return out; +} + +/** The zone's offset from UTC at an instant, in milliseconds. */ +function offsetMs(zone: string, epochMs: number): number { + const p = partsIn(zone, epochMs); + const local = Date.UTC( + p.year as number, + (p.month as number) - 1, + p.day, + p.hour, + p.minute, + p.second, + ); + return local - Math.floor(epochMs / 1000) * 1000; +} + +/** 00:00 of `day` (`YYYY-MM-DD`) in `zone`, as an RFC 3339 UTC instant. */ +export function midnightIn(day: string, zone: string): string { + const [y, m, d] = day.split("-").map(Number) as [number, number, number]; + const wall = Date.UTC(y, m - 1, d); + const first = wall - offsetMs(zone, wall); + const at = wall - offsetMs(zone, first); + return new Date(at).toISOString().replace(".000Z", "Z"); +} + +/** The calendar day of an instant in `zone` (`YYYY-MM-DD`). */ +export function dayIn(at: string, zone: string): string { + const p = partsIn(zone, Date.parse(at)); + const pad = (n: number | undefined, w = 2) => String(n).padStart(w, "0"); + return `${pad(p.year, 4)}-${pad(p.month)}-${pad(p.day)}`; +} + +const isString = (v: unknown): v is string => typeof v === "string"; + +function toSelect(value: unknown, to: FieldDefinition, ctx: ConvertContext): Conversion { + if (to.type !== "select") return fail("not a select"); + const wrap = (v: string | undefined): Conversion => + v === undefined ? ok(undefined) : ok(to.multiple ? [v] : v); + if (isString(value)) { + const t = value.trim(); + if (!t) return ok(undefined); + const lower = t.toLocaleLowerCase(ctx.defaultLocale); + const match = + to.options.find((o) => o.value === t) ?? + to.options.find((o) => o.labels[ctx.defaultLocale]?.trim().toLocaleLowerCase() === lower); + return match ? wrap(match.value) : fail(`"${t.slice(0, 40)}" is not one of the options`); + } + if (Array.isArray(value)) { + if (to.multiple) return ok(value.length ? value : undefined); + if (value.length === 0) return ok(undefined); + if (value.length === 1) return wrap(String(value[0])); + return fail(`${value.length} options chosen; a single select keeps one`); + } + return fail("not an option value"); +} + +/** One step per source type; the target decides inside it. */ +const FROM: Partial> = { + text(value, to, ctx) { + if (!isString(value)) return fail("not text"); + if (to.type === "richText") return ok(escapeMarkdown(value)); + if (to.type === "url") return ok(value.trim()); + if (to.type === "select") return toSelect(value, to, ctx); + if (to.type === "number") { + if (!/^\s*-?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\s*$/.test(value)) + return fail(`"${value.slice(0, 40)}" is not a number`); + return ok(Number(value)); + } + return fail(`text does not convert to ${to.type}`); + }, + richText(value, to) { + if (!isString(value) || to.type !== "text") return fail(`rich text converts to text only`); + return ok(markdownToPlain(value, to.multiline === true) || undefined); + }, + number(value, to) { + if (typeof value !== "number" || to.type !== "text") return fail("numbers convert to text"); + return ok(String(value)); + }, + url(value, to) { + if (!isString(value) || to.type !== "text") return fail("links convert to text"); + return ok(value); + }, + select(value, to, ctx) { + if (to.type === "select") return toSelect(value, to, ctx); + if (to.type !== "text") return fail("options convert to text or another select"); + if (isString(value)) return ok(value); + if (Array.isArray(value)) return ok(value.length ? value.join(", ") : undefined); + return fail("not an option value"); + }, + date(value, to, ctx) { + if (!isString(value) || to.type !== "datetime") return fail("dates convert to date and time"); + return ok({ at: midnightIn(value, ctx.timeZone), timeZone: ctx.timeZone }); + }, + datetime(value, to) { + const v = value as { at?: unknown; timeZone?: unknown } | null; + if (to.type !== "date" || !v || !isString(v.at) || !isString(v.timeZone)) + return fail("a date and time converts to a date"); + return ok(dayIn(v.at, v.timeZone)); + }, +}; + +/** + * Converts a value stored under a `from` field to the definition `to`, by the closed table. The + * result fits `to` (or is `undefined`: no value), or the conversion fails with a reason. + */ +export function convertFieldValue( + from: FieldType, + to: FieldDefinition, + value: unknown, + ctx: ConvertContext, +): Conversion { + let out: Conversion; + if (from === to.type && to.type !== "select") out = ok(value); + else if (!canConvert(from, to.type)) return fail(`${from} values do not convert to ${to.type}`); + else { + const step = FROM[from]; + try { + out = step ? step(value, to, ctx) : fail(`${from} values do not convert`); + } catch { + out = fail("the value could not be read"); + } + } + if (!out.ok || out.value === undefined) return out; + const issue = valueIssues(to, out.value, "v")[0]; + return issue ? fail(issue.message) : out; +} diff --git a/packages/core-ts/src/fields/definitions.ts b/packages/core-ts/src/fields/definitions.ts index 9f3fb677..dfc12aa9 100644 --- a/packages/core-ts/src/fields/definitions.ts +++ b/packages/core-ts/src/fields/definitions.ts @@ -144,7 +144,11 @@ export function fieldDefinitionChanges( for (const def of settings?.fields ?? []) { const match = next.get(def.key); if (match) out.push(...compareField(collection, def, match)); - else out.push({ collection, key: def.key, change: "removed" }); + else { + out.push({ collection, key: def.key, change: "removed" }); + // A field an operation is rewriting stays until the operation ends it. + if (def.migration) out.push({ collection, key: def.key, change: "migration" }); + } } } // A marker on a field that did not exist before is an operation's too. diff --git a/packages/core-ts/src/fields/index.ts b/packages/core-ts/src/fields/index.ts index 89189e30..9189a711 100644 --- a/packages/core-ts/src/fields/index.ts +++ b/packages/core-ts/src/fields/index.ts @@ -4,6 +4,17 @@ * express, values compiled to ADR-0030's props subset (`validateProps`), the JSON Schema agents * receive, and the values readers see (`resolveFields`). Worker-safe and Temporal-free. */ +export { + type Conversion, + type ConvertContext, + canConvert, + convertFieldValue, + dayIn, + escapeMarkdown, + FIELD_CONVERSIONS, + markdownToPlain, + midnightIn, +} from "./conversions.ts"; export { checkFieldDefinitions, type FieldDefinitionChange, @@ -15,6 +26,17 @@ export { JSON_SCHEMA_KEYWORDS, type JsonSchema, } from "./json-schema.ts"; +export { + convertContext, + type FieldMigrationKind, + isBeingRemoved, + migrationKind, + normalizeThroughMarkers, + type RewriteInput, + type RewriteResult, + rewriteFields, + type ValueChange, +} from "./migration.ts"; export { ASSET_ID_PATTERN, DATE_PATTERN, diff --git a/packages/core-ts/src/fields/migration.ts b/packages/core-ts/src/fields/migration.ts new file mode 100644 index 00000000..3a85b27d --- /dev/null +++ b/packages/core-ts/src/fields/migration.ts @@ -0,0 +1,155 @@ +/** + * RFC-0041 §6: reading and rewriting values through a field operation's `migration` marker. While + * an operation runs, the site carries the new definition with `migration: { id, fromKey | + * fromType | fromLocalized }` (no `from*` at all: the field is being removed). One function says + * what an entry's values become under the marker; the operation's dry run and rewrite, the store's + * normalization of a person's write during the run, and `resolveFields` all use it, so every + * reader agrees however many entries remain. Pure and Worker-safe. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import { type ConvertContext, convertFieldValue } from "./conversions.ts"; +import { isRecord, type SiteFields, sameJson } from "./types.ts"; +import { valueIssues } from "./value-checks.ts"; + +export type FieldMigrationKind = "rename" | "retype" | "localize" | "remove"; + +/** What a marker says the operation does (undefined: no marker). */ +export function migrationKind(def: FieldDefinition): FieldMigrationKind | undefined { + const m = def.migration; + if (!m) return undefined; + if (m.fromKey !== undefined) return "rename"; + if (m.fromType !== undefined) return "retype"; + if (m.fromLocalized !== undefined) return "localize"; + return "remove"; +} + +/** True while an operation removes the field: readers treat it as gone. */ +export const isBeingRemoved = (def: FieldDefinition): boolean => migrationKind(def) === "remove"; + +/** The conversion context of a site (`UTC` and `en` when it names none). */ +export function convertContext(site: SiteFields) { + return { + timeZone: site.timeZone ?? "UTC", + defaultLocale: site.defaultLocale ?? "en", + } satisfies ConvertContext; +} + +export interface RewriteInput { + /** The new definition, carrying the marker. */ + readonly def: FieldDefinition; + readonly values: Readonly> | undefined; + readonly convert: ConvertContext; + /** The entry is a translation whose original is in the same site. */ + readonly translation: boolean; + /** + * The entry is exactly as it was when the operation started: a type change converts its value + * even when the old value also fits the new type (text → rich text escapes). Otherwise only + * values that do not fit are converted (a person's write already used the new type). + */ + readonly pristine: boolean; + /** `"clear"` drops values that cannot be converted; otherwise they stay as stored. */ + readonly onInvalid?: "clear" | undefined; +} + +/** How one entry's value changes under a marker. */ +export interface ValueChange { + readonly kind: "moved" | "converted" | "cleared" | "removed" | "unconvertible"; + readonly before: unknown; + readonly after: unknown; + readonly reason?: string; +} + +export interface RewriteResult { + /** The new `fields`, or undefined when nothing changes. */ + readonly values: Record | undefined; + readonly change: ValueChange | undefined; +} + +const NOTHING: RewriteResult = { values: undefined, change: undefined }; + +function without(values: Readonly>, key: string) { + const { [key]: _gone, ...rest } = values; + return rest; +} + +/** A value under a type change: converted, kept, or (with `onInvalid: "clear"`) dropped. */ +function retype(input: RewriteInput, values: Readonly>): RewriteResult { + const { def, convert, pristine } = input; + const from = def.migration?.fromType; + if (!from || !Object.hasOwn(values, def.key)) return NOTHING; + const before = values[def.key]; + const fits = valueIssues(def, before, "v").length === 0; + if (fits && (!pristine || from === def.type)) return NOTHING; + const out = convertFieldValue(from, def, before, convert); + if (out.ok) { + if (sameJson(out.value, before)) return NOTHING; + const next = + out.value === undefined ? without(values, def.key) : { ...values, [def.key]: out.value }; + return { values: next, change: { kind: "converted", before, after: out.value } }; + } + if (input.onInvalid === "clear") + return { + values: without(values, def.key), + change: { kind: "cleared", before, after: undefined, reason: out.reason }, + }; + return { + values: undefined, + change: { kind: "unconvertible", before, after: undefined, reason: out.reason }, + }; +} + +/** What an entry's `fields` become under `input.def`'s marker. */ +export function rewriteFields(input: RewriteInput): RewriteResult { + const { def } = input; + const values = isRecord(input.values) ? input.values : {}; + switch (migrationKind(def)) { + case "rename": { + const from = def.migration?.fromKey as string; + if (!Object.hasOwn(values, from)) return NOTHING; + const before = values[from]; + const rest = without(values, from); + const kept = Object.hasOwn(values, def.key); + const next = kept ? rest : { ...rest, [def.key]: before }; + return { values: next, change: { kind: "moved", before, after: next[def.key] } }; + } + case "remove": + if (!Object.hasOwn(values, def.key)) return NOTHING; + return { + values: without(values, def.key), + change: { kind: "removed", before: values[def.key], after: undefined }, + }; + case "localize": + // true → false: a translation's own copy goes; the original's value is the one read. + if (def.localized || !input.translation || !Object.hasOwn(values, def.key)) return NOTHING; + return { + values: without(values, def.key), + change: { kind: "cleared", before: values[def.key], after: undefined }, + }; + case "retype": + return retype(input, values); + default: + return NOTHING; + } +} + +/** + * An entry's `fields` normalized through every marker of its collection (the store runs it on a + * person's write during an operation, so concurrent edits converge). Unconvertible values stay. + */ +export function normalizeThroughMarkers( + defs: readonly FieldDefinition[], + values: Readonly> | undefined, + context: { convert: ConvertContext; translation: boolean }, +): Record | undefined { + let current = values; + let changed = false; + for (const def of defs) { + if (!def.migration) continue; + const r = rewriteFields({ def, values: current, ...context, pristine: false }); + if (r.values) { + current = r.values; + changed = true; + } + } + return changed ? { ...current } : undefined; +} diff --git a/packages/core-ts/src/fields/resolve.ts b/packages/core-ts/src/fields/resolve.ts index 865633d5..3fcb067c 100644 --- a/packages/core-ts/src/fields/resolve.ts +++ b/packages/core-ts/src/fields/resolve.ts @@ -6,11 +6,13 @@ * - non-localized: the original's value on a translation, the entry's own otherwise; * - a language edition (an original in another site) owns all its values; * - during a field operation the `migration` marker is read through: a missing key falls back to - * `fromKey`. A value still in `fromType` that does not fit the new type resolves to `undefined` - * with an issue until the operation converts it (the closed conversion table is RFC-0041 §6's, - * applied by the operation runner). + * `fromKey`; a value still in `fromType` that does not fit the new type is converted by RFC-0041 + * §6's closed table (`conversions.ts`), and resolves to `undefined` with an issue only when the + * table cannot convert it; a field being removed is already gone. */ import type { ContentEntry, FieldDefinition } from "@cloudhal/core-contracts"; +import { convertFieldValue } from "./conversions.ts"; +import { convertContext, isBeingRemoved } from "./migration.ts"; import { type FieldIssue, fieldDefinitions, isRecord, type SiteFields } from "./types.ts"; import { valueIssues } from "./value-checks.ts"; @@ -70,6 +72,7 @@ export function resolveFields( ): ResolvedFields { const out: ResolvedFields = {}; for (const def of fieldDefinitions(site, entry.collection)) { + if (isBeingRemoved(def)) continue; const { value, fromOriginal } = pick(def, entry, original); if (value === undefined) { out[def.key] = { def, value: undefined, fromOriginal: false }; @@ -82,6 +85,17 @@ export function resolveFields( continue; } const converting = def.migration?.fromType; + const converted = converting + ? convertFieldValue(converting, def, value, convertContext(site)) + : undefined; + if (converted?.ok) { + out[def.key] = { + def, + value: converted.value, + fromOriginal: converted.value !== undefined && fromOriginal, + }; + continue; + } out[def.key] = { def, value: undefined, @@ -90,7 +104,7 @@ export function resolveFields( path: first.path, code: "field_value_unreadable", message: converting - ? `still a ${converting} value, being converted to ${def.type}` + ? `a ${converting} value that does not convert to ${def.type}` : first.message, }, }; diff --git a/packages/core-ts/src/fields/types.ts b/packages/core-ts/src/fields/types.ts index e19e15ba..d58c5064 100644 --- a/packages/core-ts/src/fields/types.ts +++ b/packages/core-ts/src/fields/types.ts @@ -9,6 +9,10 @@ export type SelectFieldDefinition = Extract /** What field code reads of a site: its collection settings. */ export interface SiteFields { collections?: Site["collections"] | undefined; + /** Conversions during a type change (`date → datetime`); `UTC` when absent. */ + timeZone?: string | undefined; + /** Conversions during a type change (`text → select` labels); `en` when absent. */ + defaultLocale?: string | undefined; } /** The nine closed types, in the contract's order. */ diff --git a/packages/core-ts/src/index.ts b/packages/core-ts/src/index.ts index 361d96f8..99558dad 100644 --- a/packages/core-ts/src/index.ts +++ b/packages/core-ts/src/index.ts @@ -33,6 +33,7 @@ export * from "./design.ts"; export * from "./domains.ts"; export * from "./editions.ts"; export * from "./errors.ts"; +export * from "./fields-operations/index.ts"; export * from "./filter.ts"; export * from "./hash.ts"; export type { NodeHold } from "./holds.ts"; diff --git a/packages/core-ts/src/store/entries.ts b/packages/core-ts/src/store/entries.ts index 54373c20..66dd6c49 100644 --- a/packages/core-ts/src/store/entries.ts +++ b/packages/core-ts/src/store/entries.ts @@ -13,7 +13,8 @@ import { entryPath } from "../permalinks.ts"; import { canonicalTerms } from "../taxonomy.ts"; import { entersCollection, planNewCollection } from "./collection-paths.ts"; import type { StoreContext } from "./context.ts"; -import { planEntryFields } from "./entry-fields.ts"; +import { normalizeEntryFields, planEntryFields } from "./entry-fields.ts"; +import { planSourceForward } from "./source-forward.ts"; import { checkLocalizedNode, checkTranslationLinks } from "./translations.ts"; import type { NodeWriteOptions } from "./types.ts"; import { checkBodyDirectives } from "./validation.ts"; @@ -57,6 +58,7 @@ export class EntryEngine { }); draft = corrections ? { ...rest, corrections } : (rest as ContentEntry); } + draft = normalizeEntryFields(site, draft); assertValid("content-entry", canonicalTerms(draft)); checkLocalizedNode("contentEntry", site, draft); await checkTranslationLinks(this.#ctx, { kind: "contentEntry", site, node: draft, stored }); @@ -100,7 +102,9 @@ export class EntryEngine { : undefined; const guard = and(prepared.guard, refs.guard, opening?.guard, fields.guard); const hash = await nodeHash(entry); - const extra = [...refs.statements(hash), ...fields.statements(hash)]; + // RFC-0041 §5: a change to non-localized values alone leaves translations fresh. + const forward = await planSourceForward(this.#ctx, site, { stored, entry, ts }); + const extra = [...refs.statements(hash), ...fields.statements(hash), ...(await forward(hash))]; const key = uniquenessKey("contentEntry", entry); const sameLocale = stored?.locale === entry.locale; const plan = await this.#ctx.tax.planPathChange(site, ts, { diff --git a/packages/core-ts/src/store/entry-fields.ts b/packages/core-ts/src/store/entry-fields.ts index 24444879..44a495ab 100644 --- a/packages/core-ts/src/store/entry-fields.ts +++ b/packages/core-ts/src/store/entry-fields.ts @@ -19,8 +19,10 @@ import type { SqlStatement } from "@cloudhal/storage"; import { checkFieldValues, checkRequiredToPublish, + convertContext, type FieldIssue, fieldDefinitions, + normalizeThroughMarkers, refuseFieldIssues, } from "../fields/index.ts"; import { sameJson } from "../fields/types.ts"; @@ -60,6 +62,22 @@ async function originalOf( return original?.siteId === entry.siteId ? original : undefined; } +/** + * RFC-0041 §6: while a field operation runs, a write goes through its marker (a renamed key moves, + * a value still in the old type converts, a removed field or a translation's copy of a field that + * stops being localized goes), so a person's edit and the operation converge. + */ +export function normalizeEntryFields(site: Site, draft: ContentEntry): ContentEntry { + const defs = fieldDefinitions(site, draft.collection); + if (!defs.some((d) => d.migration)) return draft; + const t = draft.translation; + const fields = normalizeThroughMarkers(defs, draft.fields, { + convert: convertContext(site), + translation: !!t && t.group !== draft.id && t.source?.siteId === undefined, + }); + return fields ? { ...draft, fields } : draft; +} + /** Checks the field part of an entry write and plans its asset references; throws on issues. */ export async function planEntryFields( ctx: StoreContext, @@ -143,6 +161,37 @@ function assetProblem( return undefined; } +/** + * The `fields.*` rows of `asset_refs` for an entry, rewritten: anchored on its new hash, so they + * run only when the entry's write did (rule 4). Field operations use it for every entry they move. + */ +export function fieldRefStatements( + siteId: string, + nodeId: string, + refs: readonly AssetRefUse[], + hash: string, +): SqlStatement[] { + const out: SqlStatement[] = [ + { + sql: `DELETE FROM asset_refs WHERE node_id = ? AND field LIKE 'fields.%' AND ${ANCHOR}`, + params: [nodeId, nodeId, hash], + }, + ]; + if (refs.length > 0) + out.push({ + sql: `INSERT INTO asset_refs (site_id, asset_id, node_id, node_kind, field) + SELECT ?, json_extract(value, '$.a'), ?, 'contentEntry', json_extract(value, '$.f') FROM json_each(?) WHERE ${ANCHOR}`, + params: [ + siteId, + nodeId, + JSON.stringify(refs.map((r) => ({ a: r.assetId, f: r.field }))), + nodeId, + hash, + ], + }); + return out; +} + interface RefsInput { site: Site; draft: ContentEntry; @@ -157,27 +206,7 @@ function refsPlan(ctx: StoreContext, input: RefsInput): EntryFieldsPlan { const nodeId = draft.id; const ids = [...new Set(refs.map((r) => r.assetId))]; const keptIds = [...new Set(kept.map((r) => r.assetId))]; - const statements = (hash: string): SqlStatement[] => { - const out: SqlStatement[] = [ - { - sql: `DELETE FROM asset_refs WHERE node_id = ? AND field LIKE 'fields.%' AND ${ANCHOR}`, - params: [nodeId, nodeId, hash], - }, - ]; - if (refs.length > 0) - out.push({ - sql: `INSERT INTO asset_refs (site_id, asset_id, node_id, node_kind, field) - SELECT ?, json_extract(value, '$.a'), ?, 'contentEntry', json_extract(value, '$.f') FROM json_each(?) WHERE ${ANCHOR}`, - params: [ - site.id, - nodeId, - JSON.stringify(refs.map((r) => ({ a: r.assetId, f: r.field }))), - nodeId, - hash, - ], - }); - return out; - }; + const statements = (hash: string) => fieldRefStatements(site.id, nodeId, refs, hash); if (ids.length === 0) return { guard: undefined, statements, diagnose: async () => {} }; // The files still exist, and are active unless the entry already used them (as `planRefs`). const guard: Cond = { diff --git a/packages/core-ts/src/store/source-forward.ts b/packages/core-ts/src/store/source-forward.ts new file mode 100644 index 00000000..df9cac41 --- /dev/null +++ b/packages/core-ts/src/store/source-forward.ts @@ -0,0 +1,96 @@ +/** + * RFC-0041 §5: a change to a non-localized field value of an original does not make its + * translations stale; they read the value live. Staleness compares a translation's recorded + * `translation.source.hash` with its source's current node hash (RFC-0033), so when an entry + * write changes nothing but non-localized values, the translations that were fresh (their source + * hash was the stored entry's hash) get the new hash and revision, in the same batch and guarded + * on the entry's new hash (rule 4). Node hashes stay what they are, so existing sites and their + * recorded hashes are untouched; any other change still marks translations stale, as before. + * Translations held by a release, or in other sites (editions copy values), are left alone. + */ +import { assertValid, type ContentEntry, type FieldDefinition } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import { fieldDefinitions, type SiteFields, sameJson } from "../fields/types.ts"; +import { nodeHash } from "../hash.ts"; +import { readHolds } from "../holds.ts"; +import { casUpdateStatement, hashIs } from "../node-sql.ts"; +import type { StoreContext } from "./context.ts"; + +/** Statements a write adds, built once the entry's new hash is known. */ +export type ForwardStatements = (hash: string) => Promise; + +const NONE: ForwardStatements = async () => []; + +/** The entry without its revision stamps and non-localized values. */ +function localizedView(entry: ContentEntry, shared: readonly FieldDefinition[]) { + const { rev: _r, updatedAt: _u, fields, ...rest } = entry; + const kept = { ...(fields ?? {}) }; + for (const def of shared) delete kept[def.key]; + return { ...rest, fields: kept }; +} + +/** True when the write changes non-localized values and nothing else. */ +export function onlySharedValuesChanged( + site: SiteFields, + stored: ContentEntry, + entry: ContentEntry, +): boolean { + if (stored.collection !== entry.collection) return false; + const shared = fieldDefinitions(site, entry.collection).filter((d) => !d.localized); + if (shared.length === 0) return false; + const changed = shared.some( + (d) => !sameJson(stored.fields?.[d.key] ?? null, entry.fields?.[d.key] ?? null), + ); + return changed && sameJson(localizedView(stored, shared), localizedView(entry, shared)); +} + +/** One translation re-pointed at its source's new revision and hash. */ +export function forwardedSource( + node: ContentEntry, + source: { rev: number; hash: string }, + ts: string, +): ContentEntry { + const t = node.translation as NonNullable; + const from = t.source as NonNullable; + return { + ...node, + translation: { ...t, source: { ...from, rev: source.rev, hash: source.hash } }, + rev: node.rev + 1, + updatedAt: ts, + }; +} + +/** The forward of fresh translations when `entry` changes only non-localized values. */ +export async function planSourceForward( + ctx: StoreContext, + site: SiteFields & { id: string }, + write: { stored: ContentEntry | undefined; entry: ContentEntry; ts: string }, +): Promise { + const { stored, entry, ts } = write; + if (!stored || !onlySharedValuesChanged(site, stored, entry)) return NONE; + const storedHash = await nodeHash(stored); + const rows = await ctx.sql.all<{ body: string }>( + `SELECT body FROM nodes WHERE site_id = ? AND kind = 'contentEntry' + AND json_extract(body, '$.translation.source.id') = ? + AND json_extract(body, '$.translation.source.hash') = ? + AND json_extract(body, '$.translation.source.siteId') IS NULL`, + [site.id, entry.id, storedHash], + ); + if (rows.length === 0) return NONE; + const translations = rows.map((r) => JSON.parse(r.body) as ContentEntry); + const held = await readHolds( + ctx.sql, + site.id, + translations.map((t) => t.id), + ); + const free = translations.filter((t) => !held.has(t.id)); + return async (hash) => { + const out: SqlStatement[] = []; + for (const t of free) { + const next = forwardedSource(t, { rev: entry.rev, hash }, ts); + assertValid("content-entry", next); + out.push(await casUpdateStatement("contentEntry", next, t.rev, hashIs(entry.id, hash))); + } + return out; + }; +} diff --git a/packages/core-ts/src/testing-field-operations.ts b/packages/core-ts/src/testing-field-operations.ts new file mode 100644 index 00000000..b358bd72 --- /dev/null +++ b/packages/core-ts/src/testing-field-operations.ts @@ -0,0 +1,380 @@ +/** + * RFC-0041 step 3 behavioural suite: field operations (`rename_field`, `change_field_type`, + * `set_field_localized`, `remove_field`) — dry run, snapshots, phases, resume, concurrent edits, + * revert, the draft-only guarantee — and §5's staleness rule (a non-localized value change leaves + * translations fresh). Called by `describeSiteGraphStore`, so it runs on every storage backend. + */ +import type { + Actor, + ContentEntry, + FieldDefinition, + Site, + Translation, +} from "@cloudhal/core-contracts"; +import { describe, expect, it } from "vitest"; +import { InvariantError } from "./errors.ts"; +import { resolveFields } from "./fields/index.ts"; +import { + continueFieldOperation, + type FieldOperation, + fieldOperationStatus, + revertFieldOperation, + runFieldOperation, +} from "./fields-operations/index.ts"; +import { makeJpeg } from "./media/fixtures.ts"; +import type { SiteGraphStore } from "./store.ts"; +import type { SuiteContext } from "./testing-taxonomy.ts"; + +const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; +const L = (label: string) => ({ en: { label } }); +const actor = { actor: person }; + +const FIELDS: [FieldDefinition, ...FieldDefinition[]] = [ + { key: "speaker", type: "text", labels: L("Speaker"), localized: true }, + { key: "seats", type: "text", labels: L("Seats"), localized: false }, + { key: "notes", type: "text", labels: L("Notes"), localized: true }, + { key: "price", type: "number", labels: L("Price"), localized: false }, + { key: "poster", type: "image", labels: L("Poster"), localized: false }, +]; + +async function code(p: Promise): Promise { + try { + await p; + } catch (e) { + if (e instanceof InvariantError) return e.code; + throw e; + } + throw new Error("expected a rejection"); +} + +export function fieldOperationsSuite(ctx: SuiteContext): void { + describe("RFC-0041 §6 field operations", () => { + let n = 0; + const store = (): SiteGraphStore => ctx.store(); + const fresh = async (site: Site) => (await store().getSite(site.id)) as Site; + const get = async (id: string) => (await store().getContentEntry(id)) as ContentEntry; + const run = (site: Site, op: FieldOperation, extra: Record = {}) => + runFieldOperation(store(), site.id, op, { ...actor, ...extra }); + + async function setup(): Promise { + n++; + const project = await store().createProject({ + name: "Ops", + slug: `ops-${n}`, + ownerId: person.id, + siteIds: [], + }); + return store().createSite({ + projectId: project.id, + name: "Ops", + slug: `ops-${n}`, + defaultLocale: "en", + locales: ["en", "tr"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + timeZone: "Europe/Istanbul", + collections: { events: { fields: FIELDS } }, + }); + } + const entry = ( + site: Site, + fields: Record, + extra: Partial = {}, + ) => + store().createContentEntry({ + siteId: site.id, + collection: "events", + slug: `e-${Math.random().toString(36).slice(2, 8)}`, + locale: "en", + status: "draft", + title: "Launch", + body: { format: "markdown", value: "Body" }, + fields, + tags: [], + author: person, + ...extra, + } as never); + async function translate(site: Site, original: ContentEntry, fields: Record) { + const member = (await store().translationGroup(site, original.id)).members.find( + (m) => m.id === original.id, + ); + const translation = { + group: original.id, + method: "human", + source: { id: original.id, locale: "en", rev: original.rev, hash: member?.hash as string }, + } as Translation; + return entry(site, fields, { locale: "tr", slug: original.slug, translation }); + } + const stale = async (site: Site, group: string, id: string) => + (await store().translationGroup(site, group)).members.find((m) => m.id === id)?.stale; + const keys = async (site: Site) => + ((await fresh(site)).collections?.events?.fields ?? []).map((d) => d.key); + + /** An original (made the group's original by its first translation) and its Turkish version. */ + async function world() { + const site = await setup(); + const e1 = await entry(site, { speaker: "Ada", seats: "120", notes: "n1", price: 10 }); + const e2 = await entry(site, { speaker: "Grace", seats: "lots" }); + const tr = await translate(site, e1, { speaker: "Ada TR", notes: "tr notes" }); + return { site, e1: await get(e1.id), e2, tr }; + } + + it("rename_field: dry run, snapshots, translations fresh, draft only, revert", async () => { + const { site, e1, tr } = await world(); + const published = await store().createSnapshot(site.id, { actor: person, reason: "publish" }); + await store().publish(await fresh(site), published.id); + const snapshots = (await store().listSnapshots(site.id)).length; + const op: FieldOperation = { + op: "rename_field", + collection: "events", + from: "speaker", + to: "presenter", + }; + const dry = await run(site, op, { dryRun: true }); + expect(dry).toMatchObject({ dryRun: true, done: false, operationId: "" }); + expect(dry.report).toMatchObject({ entries: 3, touched: 3, moved: 3, unconvertible: 0 }); + expect(dry.report.locales).toMatchObject({ en: { moved: 2 }, tr: { moved: 1 } }); + expect(dry.report.samples).toHaveLength(3); + expect((await store().listSnapshots(site.id)).length).toBe(snapshots); + expect(await keys(site)).toContain("speaker"); + + const done = await run(site, op); + expect(done).toMatchObject({ done: true, rewritten: 3, skipped: [] }); + const before = await store().getSnapshot(done.operationId); + expect(before?.label).toBe("before rename_field events.speaker"); + expect((await store().getSnapshot(done.afterSnapshotId as string))?.label).toBe( + "rename_field events.speaker", + ); + expect((await get(e1.id)).fields).toEqual({ + presenter: "Ada", + seats: "120", + notes: "n1", + price: 10, + }); + expect(await keys(site)).toEqual(["presenter", "seats", "notes", "price", "poster"]); + expect((await fresh(site)).collections?.events?.fields?.some((d) => d.migration)).toBe(false); + expect(await stale(site, e1.id, tr.id)).toBe(false); + // Nothing goes live: the published snapshot keeps the old definitions and values. + expect((await fresh(site)).publishedSnapshotId).toBe(published.id); + const live = published.nodes.find((r) => r.id === e1.id); + expect((await store().readSnapshotNode(live as never)).fields).toMatchObject({ + speaker: "Ada", + }); + expect((await fieldOperationStatus(store(), site.id, done.operationId)).state).toBe("done"); + + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back).toMatchObject({ done: true, restored: 3, edited: [] }); + expect((await get(e1.id)).fields).toMatchObject({ speaker: "Ada" }); + expect((await get(tr.id)).fields).toEqual({ speaker: "Ada TR", notes: "tr notes" }); + expect(await keys(site)).toEqual(["speaker", "seats", "notes", "price", "poster"]); + expect(await stale(site, e1.id, tr.id)).toBe(false); + expect((await fieldOperationStatus(store(), site.id, done.operationId)).state).toBe( + "reverted", + ); + expect(await code(revertFieldOperation(store(), site.id, done.operationId, actor))).toBe( + "field_operation_incomplete", + ); + }); + + it("change_field_type: unconvertible values refuse it unless onInvalid is clear", async () => { + const { site, e1, e2 } = await world(); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "seats", + type: "number", + }; + const dry = await run(site, op, { dryRun: true }); + expect(dry.report).toMatchObject({ converted: 1, unconvertible: 1 }); + expect(dry.report.refusal?.code).toBe("field_values_unconvertible"); + expect(dry.report.samples[0]).toMatchObject({ kind: "unconvertible", before: "lots" }); + const snapshots = (await store().listSnapshots(site.id)).length; + expect(await code(run(site, op))).toBe("field_values_unconvertible"); + expect((await store().listSnapshots(site.id)).length).toBe(snapshots); + expect((await fresh(site)).collections?.events?.fields?.[1]?.type).toBe("text"); + + const done = await run(site, { ...op, onInvalid: "clear" }); + expect(done.done).toBe(true); + expect(done.report).toMatchObject({ converted: 1, cleared: 1, unconvertible: 0 }); + expect((await get(e1.id)).fields?.seats).toBe(120); + expect((await get(e2.id)).fields).toEqual({ speaker: "Grace" }); + expect((await fresh(site)).collections?.events?.fields?.[1]?.type).toBe("number"); + + await revertFieldOperation(store(), site.id, done.operationId, actor); + expect((await get(e1.id)).fields?.seats).toBe("120"); + expect((await get(e2.id)).fields?.seats).toBe("lots"); + expect((await fresh(site)).collections?.events?.fields?.[1]).toMatchObject({ + key: "seats", + type: "text", + }); + }); + + it("set_field_localized: off drops translations' copies and keeps them fresh", async () => { + const { site, e1, tr } = await world(); + const off: FieldOperation = { + op: "set_field_localized", + collection: "events", + key: "notes", + localized: false, + }; + const dry = await run(site, off, { dryRun: true }); + expect(dry.report).toMatchObject({ touched: 1, cleared: 1, unchanged: 1 }); + expect(dry.report.locales).toEqual({ tr: expect.objectContaining({ cleared: 1 }) }); + const done = await run(site, off); + expect(done.done).toBe(true); + expect((await get(tr.id)).fields).toEqual({ speaker: "Ada TR" }); + expect((await get(e1.id)).fields?.notes).toBe("n1"); + expect(await stale(site, e1.id, tr.id)).toBe(false); + // The translation reads the original's value now. + const resolved = resolveFields(await fresh(site), await get(tr.id), await get(e1.id)); + expect(resolved.notes).toMatchObject({ value: "n1", fromOriginal: true }); + + await revertFieldOperation(store(), site.id, done.operationId, actor); + expect((await get(tr.id)).fields?.notes).toBe("tr notes"); + expect(await stale(site, e1.id, tr.id)).toBe(false); + // false → true rewrites nothing: translations fall back to the original's value. + const on = await run(site, { ...off, key: "seats", localized: true }); + expect(on).toMatchObject({ done: true, rewritten: 0 }); + expect((await fresh(site)).collections?.events?.fields?.[1]?.localized).toBe(true); + }); + + it("remove_field: values and their asset references go; revert brings them back", async () => { + const site = await setup(); + const { asset } = await store().ingestAsset({ + siteId: site.id, + bytes: makeJpeg({ width: 40, height: 30 }), + filename: "p.jpg", + uploadedBy: person, + source: { kind: "upload" }, + }); + const e = await entry(site, { poster: asset.id, speaker: "Ada" }); + const refs = async () => + ( + await ctx + .sql() + .all<{ field: string }>("SELECT field FROM asset_refs WHERE node_id = ?", [e.id]) + ).map((r) => r.field); + expect(await refs()).toEqual(["fields.poster"]); + const done = await run(site, { op: "remove_field", collection: "events", key: "poster" }); + expect(done.report).toMatchObject({ removed: 1, touched: 1 }); + expect(await keys(site)).not.toContain("poster"); + expect((await get(e.id)).fields).toEqual({ speaker: "Ada" }); + expect(await refs()).toEqual([]); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back.restored).toBe(1); + expect(await keys(site)).toEqual(["speaker", "seats", "notes", "price", "poster"]); + expect((await get(e.id)).fields?.poster).toBe(asset.id); + expect(await refs()).toEqual(["fields.poster"]); + }); + + it("resumes after its budget or an interruption; readers go through the marker", async () => { + const site = await setup(); + const made: ContentEntry[] = []; + for (let i = 0; i < 5; i++) made.push(await entry(site, { speaker: `S${i}` })); + const op: FieldOperation = { + op: "rename_field", + collection: "events", + from: "speaker", + to: "presenter", + }; + const part = await run(site, op, { budget: { entries: 2 } }); + expect(part).toMatchObject({ done: false, rewritten: 2 }); + const mid = await fresh(site); + const marked = mid.collections?.events?.fields?.find((d) => d.key === "presenter"); + expect(marked?.migration).toEqual({ id: part.operationId, fromKey: "speaker" }); + const last = made[4] as ContentEntry; + expect((await get(last.id)).fields).toEqual({ speaker: "S4" }); + expect(resolveFields(mid, await get(last.id)).presenter?.value).toBe("S4"); + expect(await fieldOperationStatus(store(), site.id, part.operationId)).toMatchObject({ + state: "running", + pending: 3, + }); + // One operation per collection; plain edits cannot drop the marker. + expect( + await code(run(site, { op: "remove_field", collection: "events", key: "notes" })), + ).toBe("operation_in_progress"); + const plain = { + ...mid, + collections: { + events: { + fields: FIELDS.map((d) => (d.key === "speaker" ? { ...d, key: "presenter" } : d)), + }, + }, + } as unknown as Site; + expect(await code(store().updateSite(plain))).toBe("field_change_needs_operation"); + // A person's write during the run goes through the marker. + const edited = await store().updateContentEntry({ ...(await get(last.id)), title: "Edited" }); + expect(edited.fields).toEqual({ presenter: "S4" }); + + const rest = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(rest).toMatchObject({ done: true, resumed: true }); + for (const e of made) + expect(Object.keys((await get(e.id)).fields ?? {})).toEqual(["presenter"]); + + // A power cut in phase B leaves the marker; continuing finishes the work. + const again: FieldOperation = { + op: "rename_field", + collection: "events", + from: "presenter", + to: "host", + }; + const failing = ctx.racing( + async () => { + throw new Error("power cut"); + }, + (q) => /^DELETE FROM asset_refs WHERE node_id = \? AND field LIKE 'fields/.test(q), + ); + await expect(runFieldOperation(failing, site.id, again, actor)).rejects.toThrow(/power cut/); + const cut = (await fresh(site)).collections?.events?.fields?.find((d) => d.key === "host"); + expect(cut?.migration?.fromKey).toBe("presenter"); + const resumed = await continueFieldOperation( + store(), + site.id, + cut?.migration?.id as string, + actor, + ); + expect(resumed.done).toBe(true); + expect((await get(last.id)).fields).toEqual({ host: "S4" }); + }); + + it("a concurrent edit skips that entry; the next pass converges", async () => { + const { site, e1 } = await world(); + const racing = ctx.racing( + async () => { + await store().updateContentEntry({ ...(await get(e1.id)), title: "Raced" }); + }, + (q) => /^DELETE FROM asset_refs WHERE node_id = \? AND field LIKE 'fields/.test(q), + ); + const done = await runFieldOperation( + racing, + site.id, + { op: "rename_field", collection: "events", from: "speaker", to: "presenter" }, + actor, + ); + expect(done.done).toBe(true); + const after = await get(e1.id); + expect(after.title).toBe("Raced"); + expect(after.fields).toMatchObject({ presenter: "Ada" }); + expect(after.fields).not.toHaveProperty("speaker"); + }); + + it("a change to non-localized values leaves translations fresh; localized ones do not", async () => { + const { site, e1, tr } = await world(); + expect(await stale(site, e1.id, tr.id)).toBe(false); + const priced = await store().updateContentEntry({ + ...e1, + fields: { ...e1.fields, price: 12 }, + }); + expect(await stale(site, e1.id, tr.id)).toBe(false); + const t = await get(tr.id); + expect(t.translation?.source?.rev).toBe(priced.rev); + // A localized value (or anything else) still marks the translation stale. + await store().updateContentEntry({ + ...priced, + fields: { ...priced.fields, price: 13, notes: "changed" }, + }); + expect(await stale(site, e1.id, tr.id)).toBe(true); + }); + }); +} diff --git a/packages/core-ts/src/testing.ts b/packages/core-ts/src/testing.ts index a80484a5..d6711dfe 100644 --- a/packages/core-ts/src/testing.ts +++ b/packages/core-ts/src/testing.ts @@ -9,6 +9,7 @@ import { nodeHash } from "./hash.ts"; import { SiteGraphStore } from "./store.ts"; import { assetsSuite } from "./testing-assets.ts"; import { collectionsSuite } from "./testing-collections.ts"; +import { fieldOperationsSuite } from "./testing-field-operations.ts"; import { fieldsSuite } from "./testing-fields.ts"; import { menuSuite } from "./testing-menu.ts"; import { permalinksSuite } from "./testing-permalinks.ts"; @@ -265,5 +266,6 @@ export function describeSiteGraphStore(label: string, factory: StoreFactory): vo permalinksSuite(suite); collectionsSuite(suite); fieldsSuite(suite); + fieldOperationsSuite(suite); }); } diff --git a/packages/core-ts/test/fields-conversions.test.ts b/packages/core-ts/test/fields-conversions.test.ts new file mode 100644 index 00000000..0c8da5e8 --- /dev/null +++ b/packages/core-ts/test/fields-conversions.test.ts @@ -0,0 +1,165 @@ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import { describe, expect, it } from "vitest"; +import { + canConvert, + convertFieldValue, + dayIn, + escapeMarkdown, + markdownToPlain, + midnightIn, + normalizeThroughMarkers, + rewriteFields, +} from "../src/fields/index.ts"; + +const L = { en: { label: "x" } }; +const ctx = { timeZone: "Europe/Istanbul", defaultLocale: "en" }; +const def = (over: Record) => + ({ key: "v", labels: L, localized: false, ...over }) as unknown as FieldDefinition; +const select = (multiple = false) => + def({ + type: "select", + multiple, + options: [ + { value: "talk", labels: { en: "Talk" } }, + { value: "workshop", labels: { en: "Hands-on workshop" } }, + ], + }); + +describe("RFC-0041 §6 conversion table", () => { + it("lists the closed table; a type always converts to itself", () => { + expect(canConvert("text", "richText")).toBe(true); + expect(canConvert("richText", "number")).toBe(false); + expect(canConvert("boolean", "boolean")).toBe(true); + expect(canConvert("boolean", "text")).toBe(false); + }); + + it("text ↔ rich text: escaped markup, plain words back", () => { + const escaped = convertFieldValue("text", def({ type: "richText" }), "a *b* [c] # d", ctx); + expect(escaped).toEqual({ ok: true, value: "a \\*b\\* \\[c\\] # d" }); + expect(escapeMarkdown("# title\n1. one")).toBe("\\# title\n1\\. one"); + const plain = convertFieldValue( + "richText", + def({ type: "text" }), + "# Hi\n\nSee **the** [docs](https://x.y).\n::callout{}\n", + ctx, + ); + expect(plain).toEqual({ ok: true, value: "Hi See the docs." }); + expect(markdownToPlain("a\n\nb", true)).toBe("a\nb"); + }); + + it("text → number accepts numbers only, and the definition's bounds", () => { + expect(convertFieldValue("text", def({ type: "number" }), " 12.5 ", ctx)).toEqual({ + ok: true, + value: 12.5, + }); + expect(convertFieldValue("text", def({ type: "number" }), "12 seats", ctx).ok).toBe(false); + const bounded = def({ type: "number", integer: true, maximum: 10 }); + expect(convertFieldValue("text", bounded, "11", ctx).ok).toBe(false); + expect(convertFieldValue("number", def({ type: "text" }), 7, ctx)).toEqual({ + ok: true, + value: "7", + }); + }); + + it("text → select by value or default-locale label; select ↔ select and → text", () => { + expect(convertFieldValue("text", select(), "talk", ctx)).toEqual({ ok: true, value: "talk" }); + expect(convertFieldValue("text", select(true), "hands-on WORKSHOP", ctx)).toEqual({ + ok: true, + value: ["workshop"], + }); + expect(convertFieldValue("text", select(), "panel", ctx).ok).toBe(false); + expect(convertFieldValue("select", select(true), "talk", ctx)).toEqual({ + ok: true, + value: ["talk"], + }); + expect(convertFieldValue("select", select(), ["talk"], ctx)).toEqual({ + ok: true, + value: "talk", + }); + expect(convertFieldValue("select", select(), ["talk", "workshop"], ctx).ok).toBe(false); + expect(convertFieldValue("select", select(), [], ctx)).toEqual({ ok: true, value: undefined }); + expect(convertFieldValue("select", def({ type: "text" }), ["talk", "workshop"], ctx)).toEqual({ + ok: true, + value: "talk, workshop", + }); + }); + + it("text ↔ url keeps the string when it is a link", () => { + expect(convertFieldValue("text", def({ type: "url" }), " https://x.y/a ", ctx)).toEqual({ + ok: true, + value: "https://x.y/a", + }); + expect(convertFieldValue("text", def({ type: "url" }), "javascript:alert(1)", ctx).ok).toBe( + false, + ); + expect(convertFieldValue("url", def({ type: "text" }), "/about", ctx).ok).toBe(true); + }); + + it("date → datetime at 00:00 in the site zone; datetime → the day in its own zone", () => { + expect(midnightIn("2026-11-05", "Europe/Istanbul")).toBe("2026-11-04T21:00:00Z"); + expect(midnightIn("2026-03-29", "Europe/Berlin")).toBe("2026-03-28T23:00:00Z"); + expect(convertFieldValue("date", def({ type: "datetime" }), "2026-11-05", ctx)).toEqual({ + ok: true, + value: { at: "2026-11-04T21:00:00Z", timeZone: "Europe/Istanbul" }, + }); + expect(dayIn("2026-11-04T22:30:00Z", "Europe/Istanbul")).toBe("2026-11-05"); + const at = { at: "2026-11-04T22:30:00Z", timeZone: "America/New_York" }; + expect(convertFieldValue("datetime", def({ type: "date" }), at, ctx)).toEqual({ + ok: true, + value: "2026-11-04", + }); + }); + + it("pairs outside the table do not convert", () => { + expect(convertFieldValue("boolean", def({ type: "date" }), true, ctx)).toMatchObject({ + ok: false, + }); + }); +}); + +describe("RFC-0041 §6 rewriting through a marker", () => { + const marker = { id: "snap_01HZX8K3Q9MG" }; + it("rename moves the value; the new key wins when both exist", () => { + const d = def({ type: "text", key: "speaker", migration: { ...marker, fromKey: "presenter" } }); + const base = { convert: ctx, translation: false, pristine: true }; + expect(rewriteFields({ def: d, values: { presenter: "Ada" }, ...base }).values).toEqual({ + speaker: "Ada", + }); + expect( + rewriteFields({ def: d, values: { presenter: "Ada", speaker: "Grace" }, ...base }).values, + ).toEqual({ speaker: "Grace" }); + }); + + it("a type change converts pristine values always, others only when they do not fit", () => { + const d = def({ type: "richText", migration: { ...marker, fromType: "text" } }); + const base = { def: d, values: { v: "a*b" }, convert: ctx, translation: false }; + expect(rewriteFields({ ...base, pristine: true }).values).toEqual({ v: "a\\*b" }); + expect(rewriteFields({ ...base, pristine: false }).values).toBeUndefined(); + const n = def({ type: "number", migration: { ...marker, fromType: "text" } }); + const bad = { def: n, values: { v: "lots" }, convert: ctx, translation: false, pristine: true }; + expect(rewriteFields(bad).change?.kind).toBe("unconvertible"); + expect(rewriteFields({ ...bad, onInvalid: "clear" })).toMatchObject({ + values: {}, + change: { kind: "cleared" }, + }); + }); + + it("localization off drops translations' copies; removal drops every value", () => { + const off = def({ + type: "text", + localized: false, + migration: { ...marker, fromLocalized: true }, + }); + const values = { v: "copy", other: 1 }; + expect(normalizeThroughMarkers([off], values, { convert: ctx, translation: true })).toEqual({ + other: 1, + }); + expect(normalizeThroughMarkers([off], values, { convert: ctx, translation: false })).toBe( + undefined, + ); + const gone = def({ type: "text", migration: marker }); + expect(normalizeThroughMarkers([gone], values, { convert: ctx, translation: false })).toEqual({ + other: 1, + }); + }); +}); diff --git a/packages/core-ts/test/fields-resolve.test.ts b/packages/core-ts/test/fields-resolve.test.ts index 62f849f1..ae89957d 100644 --- a/packages/core-ts/test/fields-resolve.test.ts +++ b/packages/core-ts/test/fields-resolve.test.ts @@ -98,7 +98,7 @@ describe("resolveFields", () => { expect(r.level?.value).toBeUndefined(); }); - it("reads through a migration marker: fromKey fallback, and unconverted values flagged", () => { + it("reads through a migration marker: fromKey fallback, conversion, removal", () => { const renamed: FieldDefinition = { key: "speaker", type: "text", @@ -118,10 +118,15 @@ describe("resolveFields", () => { localized: false, migration: { id: "snap_01HZX8K3Q9MG", fromType: "text" }, }; - const r = resolveFields(site([retyped]), entry({ seats: "120" })); + // RFC-0041 §6: values still in fromType are converted on read by the closed table. + expect(resolveFields(site([retyped]), entry({ seats: "120" })).seats?.value).toBe(120); + const r = resolveFields(site([retyped]), entry({ seats: "many" })); expect(r.seats?.value).toBeUndefined(); - expect(r.seats?.issue?.message).toContain("still a text value"); + expect(r.seats?.issue?.message).toContain("does not convert to number"); expect(resolveFields(site([retyped]), entry({ seats: 120 })).seats?.value).toBe(120); + // A field being removed (a marker without from*) is already gone for readers. + const removing = { ...retyped, migration: { id: "snap_01HZX8K3Q9MG" } } as FieldDefinition; + expect(resolveFields(site([removing]), entry({ seats: 120 })).seats).toBeUndefined(); }); });