From b853dc9d4ef7f64cd71d14fa9b0b561b2c4ced01 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:34:35 +1000 Subject: [PATCH 01/32] fix(rules): print the server message of a 409 or 422 deploy refusal A 422 can say that the requested rule order cannot be kept (a retry does not help), and a 409 can say that the change may already be live. Print that message as one short line. The old texts stay as the fallback when the response has no message. --- README.md | 8 +++++++ src/commands/rules.js | 16 +++++++++++++ tests/unit/commands/rules.test.mjs | 38 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/README.md b/README.md index 33f638f..eed345b 100644 --- a/README.md +++ b/README.md @@ -597,6 +597,14 @@ exits non-zero and lists each changed field (values only for matchers and weight Re-run with `--force` to overwrite those rules with the manifest. `--force` never takes over a rule this manifest did not create (a rule ID collision). +When the rules API refuses a deploy with 409 or 422 and gives a message, `quant rules` +prints that message and exits non-zero. Examples: + +- 409: the change may already be live. Reload the rules and check them. A retry is safe. +- 422: the requested rule order cannot be kept. A retry does not help. Contact support. + +Without a message, a 409 means another request is changing the rules (retry shortly). + Portal-issued template tokens expire after one year; rotate the token and update `QUANT_API_TOKEN` before expiry. diff --git a/src/commands/rules.js b/src/commands/rules.js index 5c29756..2b4f01d 100644 --- a/src/commands/rules.js +++ b/src/commands/rules.js @@ -59,10 +59,26 @@ function refusalError(rules) { return new Error('Managed rules were changed outside code; nothing was written. Re-run with --force to overwrite them with this manifest.'); } +const MESSAGE_STATUSES = [409, 422]; +const MAX_MESSAGE = 300; + +// The `message` (else `error`) text of a 409 or 422 body, as one short line, or null. +function serverMessage(status, data) { + if (!MESSAGE_STATUSES.includes(status)) return null; + const text = [data?.message, data?.error].find(value => typeof value === 'string' && value.trim() !== ''); + if (text === undefined) return null; + const line = text.replace(/[\u0000-\u001f\u007f-\u009f]+/g, ' ').replace(/\s+/g, ' ').trim(); + return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE - 3)}...` : line; +} + // Never print Axios config/headers or an arbitrary upstream body containing credentials. +// A 409 or 422 message is printed: the API writes it for the user (for example, +// that the change may already be live, or that the rule order cannot be kept). function deployError(error) { const status = error.response?.status; if (status === 409 && Array.isArray(error.response?.data?.rules)) return refusalError(error.response.data.rules); + const message = serverMessage(status, error.response?.data); + if (message) return new Error(`Rules deployment failed (${status}): ${message}`); if (status === 409) return new Error('Rules are being changed by another request (409). Retry shortly.'); return new Error(`Rules deployment failed (${status || error.code || 'network error'}). Check portal URL, token scopes and project access.`); } diff --git a/tests/unit/commands/rules.test.mjs b/tests/unit/commands/rules.test.mjs index d8227d0..59685ae 100644 --- a/tests/unit/commands/rules.test.mjs +++ b/tests/unit/commands/rules.test.mjs @@ -97,4 +97,42 @@ describe('Rules deployment', () => { try { await run({ force: true }); expect.fail('must fail'); } catch (error) { expect(error.message).to.include('does not own'); } try { await run(); expect.fail('must fail'); } catch (error) { expect(error.message).to.include('Retry'); } }); + describe('409 and 422 responses', () => { + async function failure(response) { + files(); + sinon.stub(axios, 'post').rejects({ response }); + sinon.stub(console, 'log'); + try { await run(); } catch (error) { return error.message; } + throw new Error('must fail'); + } + it('keeps the refusal for a 409 with a rules list', async () => { + const message = await failure({ status: 409, data: { message: 'Some managed rules changed outside code.', rules: [{ id: 'api', status: 'drift', diff: {} }] } }); + expect(message).to.include('--force').and.not.to.include('Some managed rules'); + }); + it('prints the server message of a 409', async () => { + const text = 'The change may already be live. Reload the rules and check them; a retry is safe.'; + const message = await failure({ status: 409, data: { error: 'conflict', message: text } }); + expect(message).to.equal(`Rules deployment failed (409): ${text}`); + }); + it('keeps the retry text for a 409 without a body', async () => { + expect(await failure({ status: 409 })).to.equal('Rules are being changed by another request (409). Retry shortly.'); + }); + it('prints the server message of a 422 and does not tell the user to retry', async () => { + const text = 'The requested rule order cannot be kept. Contact support.'; + const message = await failure({ status: 422, data: { error: 'rule_order', message: text } }); + expect(message).to.equal(`Rules deployment failed (422): ${text}`); + expect(message).not.to.match(/retry/i); + }); + it('prints the error text of a 422 when the body has no message', async () => { + expect(await failure({ status: 422, data: { error: 'The rule order cannot be kept.' } })).to.equal('Rules deployment failed (422): The rule order cannot be kept.'); + }); + it('keeps the generic text for a 422 without a body', async () => { + expect(await failure({ status: 422 })).to.equal('Rules deployment failed (422). Check portal URL, token scopes and project access.'); + }); + it('prints one line of a long or multi-line message', async () => { + const message = await failure({ status: 422, data: { message: `first\nsecond\u0007${'x'.repeat(600)}` } }); + expect(message).not.to.match(/[\n\u0007]/); + expect(message.length).to.be.below(400); + }); + }); }); From 400c56efb081f75f6699645f6d853a32b436cf2d Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:38:44 +1000 Subject: [PATCH 02/32] fix(serve): keep serving when the --run command moves its dev server to the background When the --run command exits 0 and its process group is still alive, print one notice that it moved to the background instead of "stopped (exit code 0)". quant serve already stops the whole group on exit. When only the --proxy URL still answers, say that quant serve cannot stop that dev server. A non-zero exit keeps the old message. Windows has no process group, which the README now says. --- README.md | 2 +- docs/adopting-quant-serve.md | 4 + src/commands/serve.js | 50 ++++++++++++- tests/fixtures/serve-run/background.mjs | 11 +++ tests/fixtures/serve-run/dev-server.mjs | 9 +++ tests/fixtures/serve-run/fail.mjs | 2 + tests/unit/commands/serve.test.mjs | 98 +++++++++++++++++++++++++ 7 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/serve-run/background.mjs create mode 100644 tests/fixtures/serve-run/dev-server.mjs create mode 100644 tests/fixtures/serve-run/fail.mjs diff --git a/README.md b/README.md index eed345b..540e35a 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ Flags: - `--user-name` - Value for `{user_name}` in KV seed values (default: `git config user.name`, then `Owner`). - `--build` - Command to run once before the server starts. - `--watch` - Reload when files change. With `--build`, run the build again when files under `src/` or `public/` change. -- `--run` - Long-running command, for example a dev server, to start beside the server. Quant serve stops it when it stops. +- `--run` - Long-running command, for example a dev server, to start beside the server. Quant serve stops it when it stops. On macOS and Linux the command runs in its own process group, and quant serve stops the whole group. Some dev servers move to the background and the command exits 0 at once. If the process group is still alive, quant serve prints one notice, keeps serving and stops the group when it stops. If only the `--proxy` URL still answers (the dev server left the process group), quant serve says that it cannot stop that dev server. On Windows there is no process group: quant serve stops only the command, and it cannot stop a dev server that moved to the background. - `--proxy` - Dev server URL. Requests that no rule ends go to this URL. WebSocket upgrades (for example, HMR) also go to this URL, after the auth rules of the path. Function rules do not run for an upgrade. - `--geo` - Geolocation fields for functions, as a JSON object. - `--client-ip` - Client address for functions. diff --git a/docs/adopting-quant-serve.md b/docs/adopting-quant-serve.md index 2aaa3ee..1f888c6 100644 --- a/docs/adopting-quant-serve.md +++ b/docs/adopting-quant-serve.md @@ -49,6 +49,10 @@ bundles function sources into the committed function bodies. Give the dev server `--host 127.0.0.1` when it listens only on IPv6 by default, or the proxy cannot reach it. + In some environments the dev server moves to the background and the `--run` + command exits 0 at once. quant serve then prints that the command moved to the + background, keeps serving, and stops the dev server when it stops. + 3. Move the seed data into fixtures. Each store gets one file, `quant/kv-fixtures/.json`, with a JSON array of entries: diff --git a/src/commands/serve.js b/src/commands/serve.js index 1225478..49583ef 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -23,6 +23,8 @@ const POSIX = process.platform !== 'win32'; /** The signals that stop quant serve. SIGHUP comes when the terminal closes. */ export const STOP_SIGNALS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); export const RUN_GRACE_MS = 2000; +/** How long to wait for the --proxy URL after the --run command exits 0. */ +export const RUN_BACKGROUND_WAIT_MS = 5000; /** The first value that is set (not undefined, null or ''), else the fallback. */ function first(values, fallback) { @@ -229,7 +231,7 @@ function rebuild(options, print) { // On POSIX the --run command gets its own process group, so that a stop // also ends the processes that the shell starts. Its stdin is not the // terminal, because a background process group cannot read the terminal. -function startRun(command, root, print) { +function startRun(command, root, print, proxy) { const child = spawn(command, { shell: true, cwd: root, @@ -238,12 +240,16 @@ function startRun(command, root, print) { }); child.on('error', (err) => print(`The --run command could not start: ${err.message}`)); child.on('exit', (code, signal) => { - if (!child.stopping) print(`The --run command stopped (${signal || `exit code ${code}`}).`); + if (child.stopping) return; + runExitLine(child, { code, signal, proxy }).then((line) => { + if (!child.stopping) print(line); + }); }); return child; } function groupAlive(pid) { + if (!POSIX) return false; try { process.kill(-pid, 0); return true; @@ -252,6 +258,44 @@ function groupAlive(pid) { } } +async function answers(url) { + try { + await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(500) }); + return true; + } catch (_err) { + return false; + } +} + +// True when the dev server answers within waitMs (it can still be starting). +async function reachable(url, waitMs) { + const deadline = Date.now() + waitMs; + for (;;) { + if (await answers(url)) return true; + if (Date.now() >= deadline) return false; + await new Promise((resolve) => setTimeout(resolve, 200)); + } +} + +/** + * The line to print when the --run command exits by itself. Some dev servers + * move to the background and the command exits 0. When its process group is + * still alive, quant serve stops that group on exit. When only the --proxy + * URL still answers (the dev server left the group, or on Windows), quant + * serve cannot stop it. + */ +export async function runExitLine(child, { code, signal, proxy, waitMs = RUN_BACKGROUND_WAIT_MS }) { + const stopped = `The --run command stopped (${signal || `exit code ${code}`}).`; + if (signal || code !== 0) return stopped; + if (groupAlive(child.pid)) { + return 'The --run command moved to the background (exit code 0). quant serve keeps serving and stops its process group when quant serve stops.'; + } + if (proxy && await reachable(proxy, waitMs)) { + return `The --run command exited (exit code 0), but the dev server at ${proxy} still answers. quant serve keeps serving, but it cannot stop that dev server. Stop it yourself.`; + } + return stopped; +} + function signalGroup(child, signal) { try { if (POSIX) process.kill(-child.pid, signal); @@ -403,7 +447,7 @@ export async function runServe(options, { stop, print = console.log, log, kvStor const server = await startOrShowKeys(serverOptions(options, { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, ...startupSummary(server.summary, kv) }); for (const line of lines) print(line); - const child = options.run ? startRun(options.run, options.root, print) : null; + const child = options.run ? startRun(options.run, options.root, print, options.proxy) : null; watchRun(server, child); const onExit = () => killRun(child); process.once('exit', onExit); diff --git a/tests/fixtures/serve-run/background.mjs b/tests/fixtures/serve-run/background.mjs new file mode 100644 index 0000000..592530f --- /dev/null +++ b/tests/fixtures/serve-run/background.mjs @@ -0,0 +1,11 @@ +// A --run command that moves the dev server to the background and exits 0, +// as some dev servers do. With "own-group", the dev server also leaves the +// process group. +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const [port, pidFile, mode] = process.argv.slice(2); +const server = fileURLToPath(new URL('./dev-server.mjs', import.meta.url)); +const child = spawn(process.execPath, [server, port, pidFile], { detached: mode === 'own-group', stdio: 'ignore' }); +child.unref(); +process.exit(0); diff --git a/tests/fixtures/serve-run/dev-server.mjs b/tests/fixtures/serve-run/dev-server.mjs new file mode 100644 index 0000000..f99fe9d --- /dev/null +++ b/tests/fixtures/serve-run/dev-server.mjs @@ -0,0 +1,9 @@ +// A small dev server for the --run tests: it listens on the given port and +// writes its pid to the given file. +import fs from 'node:fs'; +import http from 'node:http'; + +const [port, pidFile] = process.argv.slice(2); +http.createServer((_req, res) => res.end('dev')).listen(Number(port), '127.0.0.1', () => { + fs.writeFileSync(pidFile, String(process.pid)); +}); diff --git a/tests/fixtures/serve-run/fail.mjs b/tests/fixtures/serve-run/fail.mjs new file mode 100644 index 0000000..45a4a3f --- /dev/null +++ b/tests/fixtures/serve-run/fail.mjs @@ -0,0 +1,2 @@ +// A --run command that fails at once. +process.exit(1); diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 228fe07..7f373a2 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -304,6 +304,104 @@ describe('serve command', () => { }); }); + describe('--run command that exits', function () { + this.timeout(15000); + const RUN_FIXTURES = path.join(HERE, '..', '..', 'fixtures', 'serve-run'); + + function freePort() { + return new Promise((resolve) => { + const probe = net.createServer().listen(0, '127.0.0.1', () => { + const { port } = probe.address(); + probe.close(() => resolve(port)); + }); + }); + } + + async function until(check, what, timeoutMs = 8000) { + const started = Date.now(); + while (!check()) { + if (Date.now() - started > timeoutMs) throw new Error(`timed out: ${what}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + + function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (_err) { + return false; + } + } + + // Serve the Orbit fixture with the --run command until the returned stop() is called. + function serveWith(run, proxy) { + const lines = []; + let stop; + const stopped = new Promise((resolve) => { stop = resolve; }); + const opts = resolveOptions({ dir: 'dist', port: 0, memory: true, run, proxy }, { root: ORBIT, gitUserName: noGit }); + const running = runServe(opts, { stop: stopped, print: (line) => lines.push(line), log: () => {} }); + return { lines, stop: async () => { stop(); await running; } }; + } + + function fixture(name, ...args) { + return [process.execPath, path.join(RUN_FIXTURES, name), ...args].map((part) => JSON.stringify(String(part))).join(' '); + } + + it('keeps serving when the command moves the dev server to the background, and stops it on exit', async function () { + if (process.platform === 'win32') this.skip(); + const port = await freePort(); + const pidFile = path.join(tmp, 'pid'); + const served = serveWith(fixture('background.mjs', port, pidFile), `http://127.0.0.1:${port}`); + let pid = null; + try { + await until(() => served.lines.some((line) => line.includes('moved to the background')), 'the background notice'); + await until(() => fs.existsSync(pidFile) && fs.readFileSync(pidFile, 'utf8') !== '', 'the dev server'); + pid = Number(fs.readFileSync(pidFile, 'utf8')); + const response = await fetch(`http://127.0.0.1:${port}/`); + expect(await response.text()).to.equal('dev'); + } finally { + await served.stop(); + } + await until(() => !alive(pid), 'the dev server to stop', 4000); + const notices = served.lines.filter((line) => line.startsWith('The --run command')); + expect(notices).to.have.length(1); + expect(notices[0]).to.include('moved to the background').and.not.to.include('stopped (exit code 0)'); + }); + + it('says so when the background dev server left the process group', async function () { + if (process.platform === 'win32') this.skip(); + const port = await freePort(); + const pidFile = path.join(tmp, 'pid'); + const served = serveWith(fixture('background.mjs', port, pidFile, 'own-group'), `http://127.0.0.1:${port}`); + let pid = null; + try { + await until(() => served.lines.some((line) => line.includes('cannot stop')), 'the notice'); + pid = Number(fs.readFileSync(pidFile, 'utf8')); + const notice = served.lines.find((line) => line.includes('cannot stop')); + expect(notice).to.include(`http://127.0.0.1:${port}`); + } finally { + await served.stop(); + if (pid === null && fs.existsSync(pidFile)) pid = Number(fs.readFileSync(pidFile, 'utf8')); + if (pid) process.kill(pid, 'SIGKILL'); + } + }); + + it('reports a failed command as before', async () => { + const served = serveWith(fixture('fail.mjs'), undefined); + await until(() => served.lines.some((line) => line.startsWith('The --run command')), 'the exit notice'); + await served.stop(); + expect(served.lines.filter((line) => line.startsWith('The --run command'))).to.deep.equal(['The --run command stopped (exit code 1).']); + }); + + it('reports a command that exits 0 and leaves nothing behind as stopped', async () => { + const served = serveWith(`${JSON.stringify(process.execPath)} -e "process.exit(0)"`, undefined); + await until(() => served.lines.some((line) => line.startsWith('The --run command')), 'the exit notice'); + await served.stop(); + expect(served.lines.filter((line) => line.startsWith('The --run command'))).to.deep.equal(['The --run command stopped (exit code 0).']); + }); + }); + describe('runServe', () => { function options(over = {}) { return resolveOptions({ dir: 'dist', port: 0, memory: true, ...over }, { root: ORBIT, gitUserName: noGit }); From d0a655928de3bd8a83fc5ff7c45f9814185219ac Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:39:32 +1000 Subject: [PATCH 03/32] feat(kv): quant kv set --local --secret stores the value encrypted The value is stored as a function stores it with secret: true. A function on quant serve reads it, and a list shows it masked. --- README.md | 2 ++ src/commands/kv.js | 13 +++++++-- tests/unit/commands/kv.test.mjs | 51 +++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 540e35a..994f894 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,7 @@ quant kv list notes --local --index by_author --equals author=ana quant kv get notes post:1 --local quant kv set notes post:1 '{"title":"Hello"}' --local quant kv set notes post:1 '{"title":"New"}' --local --if-version 3 +quant kv set notes api:token '"abc123"' --local --secret quant kv delete notes post:1 --local ``` @@ -310,6 +311,7 @@ Flags: - `--index` - `list`: index name. - `--equals` - `list`: `field=value` for the index. Repeat the flag for more fields. - `--if-version` - `set`: write only if the key has this version. +- `--secret` - `set`: store the value encrypted, as `set(key, value, { secret: true })` in a function does. A function reads the value as usual. `list` shows `encrypted: true` and the value `[ENCRYPTED]`. `get --local` shows the value, because the local store is yours. A key in an indexed collection cannot hold a secret (`INDEX_FIELD_ENCRYPTED`). The local store uses local keys only: do not put production secrets in it. - `--org` - Organisation (default: the `quant.json` clientid, then `local`). - `--project` - Project (default: the `quant.json` project, then `dev`). - `--meta` - Template metadata with the KV stores (default: `quant/meta.json`). diff --git a/src/commands/kv.js b/src/commands/kv.js index 99fa068..9bcafcf 100644 --- a/src/commands/kv.js +++ b/src/commands/kv.js @@ -91,13 +91,21 @@ function need(value, label) { return value; } +// The set options: ifVersion, and secret (the value is stored encrypted, as +// with `secret: true` in a function: a function reads it, a list shows it masked). +function setOptions(args) { + const options = {}; + if (args['if-version'] !== undefined) options.ifVersion = args['if-version']; + if (args.secret === true) options.secret = true; + return Object.keys(options).length > 0 ? options : undefined; +} + const OPERATIONS = { list: (store, args) => listAll(store, args), get: (store, args) => store.getEntry(need(args.key, 'The key')), async set(store, args) { const value = parseJson(need(args.value, 'The JSON value')); - const ifVersion = args['if-version']; - await store.set(need(args.key, 'The key'), value, ifVersion === undefined ? undefined : { ifVersion }); + await store.set(need(args.key, 'The key'), value, setOptions(args)); return store.getEntry(args.key); }, async delete(store, args) { @@ -172,6 +180,7 @@ export default { .option('index', { type: 'string', describe: 'list: index name' }) .option('equals', { type: 'string', array: true, describe: 'list: field=value for the index (repeat)' }) .option('if-version', { type: 'string', describe: 'set: write only if the key has this version' }) + .option('secret', { type: 'boolean', describe: 'set: store the value encrypted (a list shows it masked)' }) .option('org', { type: 'string', describe: 'Organisation (default: quant.json clientid, then local)' }) .option('project', { type: 'string', describe: 'Project (default: quant.json project, then dev)' }) .option('meta', { type: 'string', default: DEFAULT_FILES.meta, describe: 'Template metadata with KV stores' }), diff --git a/tests/unit/commands/kv.test.mjs b/tests/unit/commands/kv.test.mjs index cdd5304..d60f9a9 100644 --- a/tests/unit/commands/kv.test.mjs +++ b/tests/unit/commands/kv.test.mjs @@ -107,6 +107,57 @@ describe('kv command', () => { expect(lastJson().map((item) => item.key)).to.deep.equal(['workspace:w1:project:p1']); }); + describe('set --secret', () => { + it('stores the value encrypted: get reads it, list masks it, the state file does not hold it', async () => { + expect(await run(['--local', 'set', 'orbit', 'api:token', '"s3cr3t-value"', '--secret'])).to.equal(0); + expect(lastJson()).to.include({ key: 'api:token', value: 's3cr3t-value', encrypted: true }); + expect(await run(['--local', 'list', 'orbit', '--prefix', 'api:'])).to.equal(0); + expect(lastJson()).to.have.length(1); + expect(lastJson()[0]).to.include({ key: 'api:token', encrypted: true, value: '[ENCRYPTED]' }); + const state = fs.readFileSync(path.join(root, '.quant', 'kv', 'state.json'), 'utf8'); + expect(state).not.to.include('s3cr3t-value'); + }); + + it('stores a plain value without --secret', async () => { + expect(await run(['--local', 'set', 'orbit', 'plain', '1'])).to.equal(0); + expect(lastJson().encrypted).to.equal(false); + }); + + it('works with --if-version', async () => { + await run(['--local', 'set', 'orbit', 'a', '1']); + const { version } = lastJson(); + expect(await run(['--local', 'set', 'orbit', 'a', '2', '--secret', '--if-version', version])).to.equal(0); + expect(lastJson()).to.include({ value: 2, encrypted: true }); + }); + + it('refuses a secret in an indexed collection', async () => { + const { stores } = loadProject({ root }); + await openLocalKv({ root, org: 'local', project: 'dev', stores, indexState: 'active' }); + expect(await run(['--local', 'set', 'orbit', 'workspace:w1:project:p1', '{"workspace":"w1"}', '--secret'])).to.equal(1); + expect(err[err.length - 1]).to.match(/^INDEX_FIELD_ENCRYPTED: /); + }); + + it('a function on a running server reads the secret value', async () => { + let stop; + const stopped = new Promise((resolve) => { stop = resolve; }); + let url; + const options = resolveOptions({ dir: 'dist', port: 0 }, { root, gitUserName: () => null }); + const running = runServe(options, { stop: stopped, print: () => {}, log: () => {}, onListening: (server) => { url = server.url; } }); + await new Promise((resolve) => { + const check = () => (url ? resolve() : setTimeout(check, 20)); + check(); + }); + try { + expect(await run(['--local', 'set', 'orbit', 'workspace:studio:settings', '{"name":"Secret name"}', '--secret'])).to.equal(0); + const res = await fetch(`${url}/api/workspace`); + expect((await res.json()).settings.name).to.equal('Secret name'); + } finally { + stop(); + await running; + } + }); + }); + it('refuses an undeclared store', async () => { expect(await run(['--local', 'get', 'ghost', 'a'])).to.equal(1); expect(err).to.deep.equal(['FORBIDDEN: Access to KV store local-dev-ghost is not allowed']); From 41b39e296e5a7247dd3dcedaba5eaa4a0dba0386 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:41:06 +1000 Subject: [PATCH 04/32] feat(kv): quant/serve.json gives org, project and meta defaults quant serve and quant kv --local read optional org, project and meta from quant/serve.json, so a project without quant.json needs no flags on every call. A flag still wins, and quant/serve.json wins over quant.json. The values are validated. --- README.md | 20 ++++++++++---- src/commands/kv.js | 6 ++-- src/commands/serve.js | 21 ++++++++------ src/serve/project-config.js | 35 ++++++++++++++++++++++-- tests/unit/commands/kv.test.mjs | 25 +++++++++++++++++ tests/unit/commands/serve.test.mjs | 29 ++++++++++++++++++++ tests/unit/serve/project-config.test.mjs | 18 ++++++++++-- 7 files changed, 132 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 994f894..4ac8763 100644 --- a/README.md +++ b/README.md @@ -263,9 +263,9 @@ Flags: - `--host` - Address to listen on (default: `127.0.0.1`). Any other address makes the site and its KV data reachable on the network. - `--functions` - Functions manifest (default: `quant/edge-functions.json`). - `--rules` - Rules manifest (default: `quant/edge-rules.json`). -- `--meta` - Template metadata with the KV stores (default: `quant/meta.json`). +- `--meta` - Template metadata with the KV stores (default: `meta` in `quant/serve.json`, then `quant/meta.json`). - `--fixtures` - Directory of KV fixtures (default: `quant/kv-fixtures`). -- `--org` - Organisation (default: the `quant.json` clientid, then `local`). +- `--org` - Organisation (default: `org` in `quant/serve.json`, the `quant.json` clientid, then `local`). - `--project` - Project (default: the `quant.json` project, then `dev`). - `--domain` - Host for the domain match of rules (default: the Host of the request). - `--user-name` - Value for `{user_name}` in KV seed values (default: `git config user.name`, then `Owner`). @@ -312,9 +312,11 @@ Flags: - `--equals` - `list`: `field=value` for the index. Repeat the flag for more fields. - `--if-version` - `set`: write only if the key has this version. - `--secret` - `set`: store the value encrypted, as `set(key, value, { secret: true })` in a function does. A function reads the value as usual. `list` shows `encrypted: true` and the value `[ENCRYPTED]`. `get --local` shows the value, because the local store is yours. A key in an indexed collection cannot hold a secret (`INDEX_FIELD_ENCRYPTED`). The local store uses local keys only: do not put production secrets in it. -- `--org` - Organisation (default: the `quant.json` clientid, then `local`). -- `--project` - Project (default: the `quant.json` project, then `dev`). -- `--meta` - Template metadata with the KV stores (default: `quant/meta.json`). +- `--org` - Organisation (default: `org` in `quant/serve.json`, the `quant.json` clientid, then `local`). +- `--project` - Project (default: `project` in `quant/serve.json`, the `quant.json` project, then `dev`). +- `--meta` - Template metadata with the KV stores (default: `meta` in `quant/serve.json`, then `quant/meta.json`). + +To set these defaults one time for a project, put them in `quant/serve.json` (see [Local project settings](#local-project-settings-quantservejson)). ## Local ledgers @@ -448,6 +450,14 @@ Each field is optional. - `globalHeaders` - Headers for every response from the site: pages, files, function responses, the 404 page and the basic authentication 401. A header that a function sets replaces the global header of the same name. `quant-server` and `age` cannot be set. - `basicAuth` - With a username and a password, every path needs these credentials. A request without them gets 401 with `WWW-Authenticate: Basic`. The check applies before the rules, so functions and redirects need the credentials too. WebSocket upgrades also need them. The `/_quant_serve/` control routes do not need them: they are local only and need the `x-quant-serve: 1` header (see [Safety](#safety)). - `trailingSlash` - `"redirect"`: a path that ends with `/` (except `/`) gets a 301 to the path without the trailing slashes, with the query string. A path that a rule answers is not redirected. `"none"` (the default): no redirect. +- `org`, `project` - The organisation and the project for the local KV store ids, for `quant serve` and `quant kv --local`. Use them when the project has no `quant.json`. Each value is 1 to 100 letters, digits, `.`, `_` or `-`. +- `meta` - The template metadata file with the KV stores (default: `quant/meta.json`). The path is relative to the project directory and must stay in it. + +For `org`, `project` and `meta`, the order is: the flag, then `quant/serve.json`, then `quant.json` (org and project only), then the default (`local`, `dev`, `quant/meta.json`). `quant serve` reads these three fields only at start-up, also with `--watch`. + +```json +{ "org": "acme", "project": "marketing-site" } +``` The order is the order of the edge: global headers, basic authentication, rules, the trailing-slash redirect, then static content. diff --git a/src/commands/kv.js b/src/commands/kv.js index 9bcafcf..8ea6b8e 100644 --- a/src/commands/kv.js +++ b/src/commands/kv.js @@ -181,9 +181,9 @@ export default { .option('equals', { type: 'string', array: true, describe: 'list: field=value for the index (repeat)' }) .option('if-version', { type: 'string', describe: 'set: write only if the key has this version' }) .option('secret', { type: 'boolean', describe: 'set: store the value encrypted (a list shows it masked)' }) - .option('org', { type: 'string', describe: 'Organisation (default: quant.json clientid, then local)' }) - .option('project', { type: 'string', describe: 'Project (default: quant.json project, then dev)' }) - .option('meta', { type: 'string', default: DEFAULT_FILES.meta, describe: 'Template metadata with KV stores' }), + .option('org', { type: 'string', describe: 'Organisation (default: quant/serve.json org, quant.json clientid, then local)' }) + .option('project', { type: 'string', describe: 'Project (default: quant/serve.json project, quant.json project, then dev)' }) + .option('meta', { type: 'string', describe: `Template metadata with KV stores (default: quant/serve.json meta, then ${DEFAULT_FILES.meta})` }), promptArgs, async handler(args) { const code = await runKv(args); diff --git a/src/commands/serve.js b/src/commands/serve.js index 49583ef..fef08b0 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -14,6 +14,7 @@ import { startServer } from '../serve/server.js'; import { DEFAULT_FILES, loadProject } from '../serve/project.js'; import { openLocalKv } from '../serve/kv/wire.js'; import { LEDGERS_FILE, readLedgerFile } from '../serve/kv/ledgers.js'; +import { readServeConfig } from '../serve/project-config.js'; const DEFAULTS = Object.freeze({ dir: 'dist', port: 8787, host: '127.0.0.1', org: 'local', project: 'dev', userName: 'Owner' }); const LATER_RELEASE = ['pull-rules', 'kv-remote']; @@ -98,23 +99,23 @@ function clockOf(value) { return clock; } -function siteOptions(args) { +function siteOptions(args, local) { return { dir: first([args.dir], DEFAULTS.dir), port: portOf(args.port), host: first([args.host], DEFAULTS.host), functions: first([args.functions], DEFAULT_FILES.functions), rules: first([args.rules], DEFAULT_FILES.rules), - meta: first([args.meta], DEFAULT_FILES.meta), + meta: first([args.meta, local.meta], DEFAULT_FILES.meta), fixtures: first([args.fixtures], DEFAULT_FILES.fixtures) }; } -function identityOptions(args, root, gitUserName) { +function identityOptions(args, root, gitUserName, local) { const saved = readQuantJson(root); return { - org: first([args.org, args.clientid, saved.clientid], DEFAULTS.org), - project: first([args.project, saved.project], DEFAULTS.project), + org: first([args.org, args.clientid, local.org, saved.clientid], DEFAULTS.org), + project: first([args.project, local.project, saved.project], DEFAULTS.project), domain: first([args.domain], undefined), userName: first([args['user-name'], gitUserName(root)], DEFAULTS.userName) }; @@ -150,10 +151,12 @@ function devOptions(args) { */ export function resolveOptions(args, { root = process.cwd(), gitUserName = readGitUserName } = {}) { refuseLaterFlags(args); + // The org, project and meta defaults of quant/serve.json (a flag wins). + const local = readServeConfig(root).defaults; return { root, - ...siteOptions(args), - ...identityOptions(args, root, gitUserName), + ...siteOptions(args, local), + ...identityOptions(args, root, gitUserName, local), ...kvOptions(args), ...devOptions(args) }; @@ -513,9 +516,9 @@ export default { .option('host', { type: 'string', default: DEFAULTS.host, describe: 'Address to listen on' }) .option('functions', { type: 'string', default: DEFAULT_FILES.functions, describe: 'Functions manifest' }) .option('rules', { type: 'string', default: DEFAULT_FILES.rules, describe: 'Rules manifest' }) - .option('meta', { type: 'string', default: DEFAULT_FILES.meta, describe: 'Template metadata with KV stores' }) + .option('meta', { type: 'string', describe: `Template metadata with KV stores (default: quant/serve.json meta, then ${DEFAULT_FILES.meta})` }) .option('fixtures', { type: 'string', default: DEFAULT_FILES.fixtures, describe: 'KV fixtures directory' }) - .option('org', { type: 'string', describe: 'Organisation (default: quant.json clientid, then local)' }) + .option('org', { type: 'string', describe: 'Organisation (default: quant/serve.json org, quant.json clientid, then local)' }) .option('domain', { type: 'string', describe: 'Host for rule domain matching (default: the request Host)' }) .option('user-name', { type: 'string', describe: '{user_name} value (default: git config user.name, then Owner)' }) .option('reset', { type: 'boolean', describe: 'Delete the local KV state and seed again' }) diff --git a/src/serve/project-config.js b/src/serve/project-config.js index ae2dd08..3adf43e 100644 --- a/src/serve/project-config.js +++ b/src/serve/project-config.js @@ -6,6 +6,9 @@ * * The request order is the order of the Quant edge: global headers, basic * authentication, rules, the trailing-slash redirect, then static content. + * + * The file can also give the defaults of org, project and meta for + * `quant serve` and `quant kv --local`. A flag wins over the file. */ import crypto from 'node:crypto'; @@ -14,7 +17,9 @@ import path from 'node:path'; export const SERVE_CONFIG = 'quant/serve.json'; -const FIELDS = ['globalHeaders', 'basicAuth', 'trailingSlash']; +const IDENTITY_FIELDS = ['org', 'project', 'meta']; +const FIELDS = ['globalHeaders', 'basicAuth', 'trailingSlash', ...IDENTITY_FIELDS]; +const SLUG = /^[A-Za-z0-9._-]{1,100}$/; const TRAILING_SLASH = ['redirect', 'none']; const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; const HEADER_VALUE = /^[\t\x20-\x7e]*$/; @@ -55,9 +60,34 @@ function trailingSlash(value, source) { return value; } +function slug(name, value, source) { + if (typeof value !== 'string' || !SLUG.test(value)) { + throw new Error(`${source}: ${name} must be letters, digits, ".", "_" or "-" (1 to 100 characters).`); + } + return value; +} + +function metaFile(_name, value, source) { + const inside = isText(value) && !path.isAbsolute(value) && !path.normalize(value).startsWith('..'); + if (!inside) throw new Error(`${source}: meta must be a path to a file in the project, relative to the project directory.`); + return value; +} + +const IDENTITY_CHECKS = { org: slug, project: slug, meta: metaFile }; + +// The org, project and meta defaults that the document gives. +function identityDefaults(document, source) { + const defaults = {}; + for (const name of IDENTITY_FIELDS.filter((field) => field in document)) { + defaults[name] = IDENTITY_CHECKS[name](name, document[name], source); + } + return defaults; +} + /** * Check a parsed quant/serve.json document. - * @returns {{ globalHeaders: Array<[string, string]>, basicAuth: string|null, trailingSlash: string, warnings: string[] }} + * @returns {{ globalHeaders: Array<[string, string]>, basicAuth: string|null, trailingSlash: string, + * defaults: { org?: string, project?: string, meta?: string }, warnings: string[] }} */ export function parseServeConfig(document, source = SERVE_CONFIG) { if (!isObject(document)) throw new Error(`${source} must be a JSON object.`); @@ -68,6 +98,7 @@ export function parseServeConfig(document, source = SERVE_CONFIG) { globalHeaders: 'globalHeaders' in document ? globalHeaders(document.globalHeaders, source, warnings) : [], basicAuth: 'basicAuth' in document ? basicAuth(document.basicAuth, source) : null, trailingSlash: 'trailingSlash' in document ? trailingSlash(document.trailingSlash, source) : 'none', + defaults: identityDefaults(document, source), warnings }; } diff --git a/tests/unit/commands/kv.test.mjs b/tests/unit/commands/kv.test.mjs index d60f9a9..12205e0 100644 --- a/tests/unit/commands/kv.test.mjs +++ b/tests/unit/commands/kv.test.mjs @@ -169,6 +169,31 @@ describe('kv command', () => { expect(JSON.stringify(state)).to.include('acme-site-orbit'); }); + describe('defaults from quant/serve.json', () => { + const serveJson = (document) => fs.writeFileSync(path.join(root, 'quant', 'serve.json'), JSON.stringify(document)); + const state = () => fs.readFileSync(path.join(root, '.quant', 'kv', 'state.json'), 'utf8'); + + it('uses org, project and meta from quant/serve.json', async () => { + fs.mkdirSync(path.join(root, 'config')); + fs.renameSync(path.join(root, 'quant', 'meta.json'), path.join(root, 'config', 'meta.json')); + serveJson({ org: 'acme', project: 'site', meta: 'config/meta.json' }); + expect(await run(['--local', 'set', 'orbit', 'a', '1'])).to.equal(0); + expect(state()).to.include('acme-site-orbit'); + }); + + it('lets the flags win', async () => { + serveJson({ org: 'acme', project: 'site', meta: 'missing.json' }); + expect(await run(['--local', '--org', 'other', '--project', 'web', '--meta', 'quant/meta.json', 'set', 'orbit', 'a', '1'])).to.equal(0); + expect(state()).to.include('other-web-orbit').and.not.to.include('acme-site-orbit'); + }); + + it('refuses an invalid quant/serve.json', async () => { + serveJson({ project: '' }); + expect(await run(['--local', 'get', 'orbit', 'a'])).to.equal(1); + expect(err[0]).to.match(/^quant\/serve\.json: project must be/); + }); + }); + it('leaves a BUILDING index BUILDING', async () => { const { stores } = loadProject({ root }); const first = await openLocalKv({ root, org: 'local', project: 'dev', stores, indexState: 'BUILDING' }); diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 7f373a2..9b2f3ca 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -106,6 +106,35 @@ describe('serve command', () => { expect(JSON.stringify(options)).not.to.include('secret'); }); + describe('defaults from quant/serve.json', () => { + function serveJson(document) { + fs.mkdirSync(path.join(tmp, 'quant'), { recursive: true }); + fs.writeFileSync(path.join(tmp, 'quant', 'serve.json'), JSON.stringify(document)); + } + + it('reads org, project and meta from quant/serve.json', () => { + serveJson({ org: 'acme', project: 'site', meta: 'config/meta.json' }); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit })).to.include({ org: 'acme', project: 'site', meta: 'config/meta.json' }); + }); + + it('lets the flags win', () => { + serveJson({ org: 'acme', project: 'site', meta: 'config/meta.json' }); + const options = resolveOptions(parse(['--org', 'flag-org', '--project', 'flag-project', '--meta', 'flag/meta.json']), { root: tmp, gitUserName: noGit }); + expect(options).to.include({ org: 'flag-org', project: 'flag-project', meta: 'flag/meta.json' }); + }); + + it('wins over quant.json', () => { + fs.writeFileSync(path.join(tmp, 'quant.json'), JSON.stringify({ clientid: 'saved', project: 'saved-project' })); + serveJson({ org: 'acme' }); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit })).to.include({ org: 'acme', project: 'saved-project' }); + }); + + it('refuses an invalid value', () => { + serveJson({ org: 'a b' }); + expect(() => resolveOptions(parse([]), { root: tmp, gitUserName: noGit })).to.throw('quant/serve.json: org must be'); + }); + }); + it('reads quant.json softly', () => { fs.writeFileSync(path.join(tmp, 'quant.json'), '{ broken'); const options = resolveOptions(parse([]), { root: tmp, gitUserName: noGit }); diff --git a/tests/unit/serve/project-config.test.mjs b/tests/unit/serve/project-config.test.mjs index 4311cb9..ae08559 100644 --- a/tests/unit/serve/project-config.test.mjs +++ b/tests/unit/serve/project-config.test.mjs @@ -18,7 +18,7 @@ function settings(document) { describe('serve project settings (quant/serve.json)', () => { describe('parseServeConfig', () => { it('gives the defaults for an empty document', () => { - expect(settings({})).to.deep.equal({ globalHeaders: [], basicAuth: null, trailingSlash: 'none', warnings: [] }); + expect(settings({})).to.deep.equal({ globalHeaders: [], basicAuth: null, trailingSlash: 'none', defaults: {}, warnings: [] }); }); it('reads every field', () => { @@ -28,6 +28,12 @@ describe('serve project settings (quant/serve.json)', () => { expect(result.trailingSlash).to.equal('redirect'); }); + it('reads the org, project and meta defaults for local KV commands', () => { + const result = settings({ org: 'acme', project: 'site-1', meta: 'config/meta.json' }); + expect(result.defaults).to.deep.equal({ org: 'acme', project: 'site-1', meta: 'config/meta.json' }); + expect(settings({ project: 'p' }).defaults).to.deep.equal({ project: 'p' }); + }); + it('skips the headers that a project cannot set, with a warning', () => { const result = settings({ globalHeaders: { 'quant-server': 'x', Age: '1', 'x-a': '1' } }); expect(result.globalHeaders).to.deep.equal([['x-a', '1']]); @@ -38,7 +44,7 @@ describe('serve project settings (quant/serve.json)', () => { it('refuses a document with the wrong shape, with a clear message', () => { const cases = [ [[], 'quant/serve.json must be a JSON object.'], - [{ headers: {} }, 'quant/serve.json: headers is not supported (supported: globalHeaders, basicAuth, trailingSlash).'], + [{ headers: {} }, 'quant/serve.json: headers is not supported (supported: globalHeaders, basicAuth, trailingSlash, org, project, meta).'], [{ globalHeaders: [] }, 'quant/serve.json: globalHeaders must be an object of header names and values.'], [{ globalHeaders: { 'bad name': 'x' } }, 'quant/serve.json: globalHeaders: "bad name" is not a valid header name.'], [{ globalHeaders: { 'x-a': 1 } }, 'quant/serve.json: globalHeaders.x-a must be printable ASCII text on one line.'], @@ -46,7 +52,13 @@ describe('serve project settings (quant/serve.json)', () => { [{ basicAuth: { username: 'u' } }, /basicAuth must be \{ "username": "\.\.\.", "password": "\.\.\." \}/], [{ basicAuth: { username: 'u', password: '' } }, /basicAuth must be/], [{ basicAuth: { username: 'u:v', password: 'p' } }, 'quant/serve.json: basicAuth.username cannot contain ":".'], - [{ trailingSlash: 'preserve' }, 'quant/serve.json: trailingSlash must be "redirect" or "none".'] + [{ trailingSlash: 'preserve' }, 'quant/serve.json: trailingSlash must be "redirect" or "none".'], + [{ org: '' }, 'quant/serve.json: org must be letters, digits, ".", "_" or "-" (1 to 100 characters).'], + [{ project: 'my site' }, 'quant/serve.json: project must be letters, digits, ".", "_" or "-" (1 to 100 characters).'], + [{ org: 7 }, /org must be letters/], + [{ meta: '' }, 'quant/serve.json: meta must be a path to a file in the project, relative to the project directory.'], + [{ meta: '/etc/meta.json' }, /meta must be a path to a file in the project/], + [{ meta: '../other/meta.json' }, /meta must be a path to a file in the project/] ]; for (const [document, message] of cases) expect(() => settings(document), JSON.stringify(document)).to.throw(message); }); From cf49cfecf664a1d286ca1d3c931be5b83af43693 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:43:38 +1000 Subject: [PATCH 05/32] feat(serve): warn when quant/serve.json and quant.json name a different org or project The quant/serve.json value is used. quant serve prints the warning in the start-up output, and quant kv --local prints it to stderr. --- README.md | 2 +- src/commands/kv.js | 3 ++- src/commands/serve.js | 18 ++++++++++++++++- tests/unit/commands/kv.test.mjs | 8 ++++++++ tests/unit/commands/serve.test.mjs | 31 ++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4ac8763..3b3f75e 100644 --- a/README.md +++ b/README.md @@ -453,7 +453,7 @@ Each field is optional. - `org`, `project` - The organisation and the project for the local KV store ids, for `quant serve` and `quant kv --local`. Use them when the project has no `quant.json`. Each value is 1 to 100 letters, digits, `.`, `_` or `-`. - `meta` - The template metadata file with the KV stores (default: `quant/meta.json`). The path is relative to the project directory and must stay in it. -For `org`, `project` and `meta`, the order is: the flag, then `quant/serve.json`, then `quant.json` (org and project only), then the default (`local`, `dev`, `quant/meta.json`). `quant serve` reads these three fields only at start-up, also with `--watch`. +For `org`, `project` and `meta`, the order is: the flag, then `quant/serve.json`, then `quant.json` (org and project only), then the default (`local`, `dev`, `quant/meta.json`). `quant serve` reads these three fields only at start-up, also with `--watch`. If `quant/serve.json` and `quant.json` set a different org (or project) and no flag chooses, `quant serve` and `quant kv --local` print one warning line that names the value they use. ```json { "org": "acme", "project": "marketing-site" } diff --git a/src/commands/kv.js b/src/commands/kv.js index 8ea6b8e..086666c 100644 --- a/src/commands/kv.js +++ b/src/commands/kv.js @@ -128,7 +128,8 @@ async function execute(args, root, error) { const stores = readStores(root, options.meta).map((store) => ({ ...store, indexes: [] })); const local = await openLocalKv({ root, org: options.org, project: options.project, stores }); // The warnings come before the operation, so a failed operation keeps them. - for (const warning of [...local.warnings, ...pendingLedgerWarnings(local.backend, local.backend.storeIds())]) error(warning); + const warnings = [...options.configWarnings, ...local.warnings, ...pendingLedgerWarnings(local.backend, local.backend.storeIds())]; + for (const warning of warnings) error(warning); return OPERATIONS[args.action](local.kvStore.open(id), args); } diff --git a/src/commands/serve.js b/src/commands/serve.js index fef08b0..a486e9e 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -111,9 +111,24 @@ function siteOptions(args, local) { }; } +// A warning for each of org and project that quant/serve.json and quant.json +// set to different values, when no flag chooses: quant/serve.json wins. +function precedenceWarnings(args, local, saved) { + const fields = [ + { name: 'org', flag: first([args.org, args.clientid], undefined), saved: saved.clientid, field: 'clientid' }, + { name: 'project', flag: first([args.project], undefined), saved: saved.project, field: 'project' } + ]; + return fields + .filter((item) => item.flag === undefined && local[item.name] !== undefined && first([item.saved], undefined) !== undefined && + String(item.saved) !== local[item.name]) + .map((item) => `quant/serve.json ${item.name} "${local[item.name]}" is used, not quant.json ${item.field} "${item.saved}": ` + + `quant/serve.json wins over quant.json. Pass --${item.name} to choose.`); +} + function identityOptions(args, root, gitUserName, local) { const saved = readQuantJson(root); return { + configWarnings: precedenceWarnings(args, local, saved), org: first([args.org, args.clientid, local.org, saved.clientid], DEFAULTS.org), project: first([args.project, local.project, saved.project], DEFAULTS.project), domain: first([args.domain], undefined), @@ -448,7 +463,8 @@ export async function runServe(options, { stop, print = console.log, log, kvStor if (options.build) runBuild(options.build, options.root); const kv = await kvFor(options, kvStore, log, print); const server = await startOrShowKeys(serverOptions(options, { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); - const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, ...startupSummary(server.summary, kv) }); + const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...server.summary.warnings] }; + const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, ...startupSummary(summary, kv) }); for (const line of lines) print(line); const child = options.run ? startRun(options.run, options.root, print, options.proxy) : null; watchRun(server, child); diff --git a/tests/unit/commands/kv.test.mjs b/tests/unit/commands/kv.test.mjs index 12205e0..cd7d62c 100644 --- a/tests/unit/commands/kv.test.mjs +++ b/tests/unit/commands/kv.test.mjs @@ -187,6 +187,14 @@ describe('kv command', () => { expect(state()).to.include('other-web-orbit').and.not.to.include('acme-site-orbit'); }); + it('warns once when quant/serve.json and quant.json name a different org', async () => { + fs.writeFileSync(path.join(root, 'quant.json'), JSON.stringify({ clientid: 'saved', project: 'dev' })); + serveJson({ org: 'acme' }); + expect(await run(['--local', 'set', 'orbit', 'a', '1'])).to.equal(0); + expect(err).to.deep.equal(['quant/serve.json org "acme" is used, not quant.json clientid "saved": quant/serve.json wins over quant.json. Pass --org to choose.']); + expect(state()).to.include('acme-dev-orbit'); + }); + it('refuses an invalid quant/serve.json', async () => { serveJson({ project: '' }); expect(await run(['--local', 'get', 'orbit', 'a'])).to.equal(1); diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 9b2f3ca..2ed5fb3 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -129,6 +129,37 @@ describe('serve command', () => { expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit })).to.include({ org: 'acme', project: 'saved-project' }); }); + it('warns when quant/serve.json and quant.json set a different org or project', () => { + fs.writeFileSync(path.join(tmp, 'quant.json'), JSON.stringify({ clientid: 'saved', project: 'saved-project' })); + serveJson({ org: 'acme', project: 'saved-project' }); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit }).configWarnings).to.deep.equal([ + 'quant/serve.json org "acme" is used, not quant.json clientid "saved": quant/serve.json wins over quant.json. Pass --org to choose.' + ]); + serveJson({ project: 'site' }); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit }).configWarnings).to.deep.equal([ + 'quant/serve.json project "site" is used, not quant.json project "saved-project": quant/serve.json wins over quant.json. Pass --project to choose.' + ]); + }); + + it('does not warn when a flag chooses, or when the values are the same', () => { + fs.writeFileSync(path.join(tmp, 'quant.json'), JSON.stringify({ clientid: 'saved', project: 'p' })); + serveJson({ org: 'acme', project: 'p' }); + expect(resolveOptions(parse(['--org', 'x']), { root: tmp, gitUserName: noGit }).configWarnings).to.deep.equal([]); + serveJson({ org: 'saved' }); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit }).configWarnings).to.deep.equal([]); + }); + + it('prints the warning in the start-up output', async () => { + const root = path.join(tmp, 'project'); + fs.cpSync(ORBIT, root, { recursive: true, filter: (src) => !src.includes(`${path.sep}.quant`) }); + fs.writeFileSync(path.join(root, 'quant.json'), JSON.stringify({ clientid: 'saved' })); + fs.writeFileSync(path.join(root, 'quant', 'serve.json'), JSON.stringify({ org: 'acme' })); + const opts = resolveOptions({ dir: 'dist', port: 0, memory: true }, { root, gitUserName: noGit }); + await runServe(opts, { stop: Promise.resolve(), print: (line) => output.push(line), log: () => {} }); + const warnings = output.filter((line) => line.includes('wins over quant.json')); + expect(warnings).to.have.length(1); + }); + it('refuses an invalid value', () => { serveJson({ org: 'a b' }); expect(() => resolveOptions(parse([]), { root: tmp, gitUserName: noGit })).to.throw('quant/serve.json: org must be'); From 529da64c705143dbae51742b38a07e8b254a0ec9 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:43:58 +1000 Subject: [PATCH 06/32] feat(kv): the kv set menu asks whether to store the value as a secret A yes/no question after the JSON value, default no. --- src/commands/kv.js | 39 +++++++++++++++++++++++---------- tests/unit/commands/kv.test.mjs | 37 ++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/commands/kv.js b/src/commands/kv.js index 086666c..f7c4446 100644 --- a/src/commands/kv.js +++ b/src/commands/kv.js @@ -6,7 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; -import { select, text, isCancel } from '@clack/prompts'; +import { select, text, confirm, isCancel } from '@clack/prompts'; import { openLocalKv } from '../serve/kv/wire.js'; import { pendingLedgerWarnings } from '../serve/kv/ledgers.js'; import { DEFAULT_FILES } from '../serve/project.js'; @@ -153,18 +153,33 @@ export async function runKv(args, { root = process.cwd(), print = console.log, e } } -// Ask for the action, the store and what the action needs. Null on cancel. -async function promptArgs() { - const action = await select({ message: 'Operation', options: ACTIONS.map((value) => ({ value, label: value })) }); - if (isCancel(action)) return null; - const store = await text({ message: 'Store name (from quant/meta.json)' }); - if (isCancel(store)) return null; +const PROMPTS = { select, text, confirm, isCancel }; + +// set: the JSON value and the secret question (default no). Null on cancel. +async function askValue(prompts) { + const value = await prompts.text({ message: 'JSON value' }); + if (prompts.isCancel(value)) return null; + const secret = await prompts.confirm({ message: 'Store the value as a secret (encrypted)?', initialValue: false }); + return prompts.isCancel(secret) ? null : { value, secret: secret === true }; +} + +/** + * The menu: ask for the action, the store and what the action needs. + * @param {{ select, text, confirm, isCancel }} [prompts] The prompt functions (for tests). + * @returns {Promise} The args, or null on cancel. + */ +export async function askKv(prompts = PROMPTS) { + const action = await prompts.select({ message: 'Operation', options: ACTIONS.map((value) => ({ value, label: value })) }); + if (prompts.isCancel(action)) return null; + const store = await prompts.text({ message: 'Store name (from quant/meta.json)' }); + if (prompts.isCancel(store)) return null; const args = { local: true, action, store }; if (action === 'list') return args; - const key = await text({ message: 'Key' }); - if (isCancel(key)) return null; - const value = action === 'set' ? await text({ message: 'JSON value' }) : undefined; - return isCancel(value) ? null : { ...args, key, value }; + const key = await prompts.text({ message: 'Key' }); + if (prompts.isCancel(key)) return null; + if (action !== 'set') return { ...args, key, value: undefined }; + const answers = await askValue(prompts); + return answers === null ? null : { ...args, key, ...answers }; } export default { @@ -185,7 +200,7 @@ export default { .option('org', { type: 'string', describe: 'Organisation (default: quant/serve.json org, quant.json clientid, then local)' }) .option('project', { type: 'string', describe: 'Project (default: quant/serve.json project, quant.json project, then dev)' }) .option('meta', { type: 'string', describe: `Template metadata with KV stores (default: quant/serve.json meta, then ${DEFAULT_FILES.meta})` }), - promptArgs, + promptArgs: () => askKv(), async handler(args) { const code = await runKv(args); if (code !== 0) process.exit(code); diff --git a/tests/unit/commands/kv.test.mjs b/tests/unit/commands/kv.test.mjs index cd7d62c..42a464c 100644 --- a/tests/unit/commands/kv.test.mjs +++ b/tests/unit/commands/kv.test.mjs @@ -8,7 +8,7 @@ import yargs from 'yargs'; import { createVirtualClock } from '@quantcdn/edge-kv'; import { openLocalKv } from '../../../src/serve/kv/wire.js'; import { loadProject } from '../../../src/serve/project.js'; -import kv, { runKv } from '../../../src/commands/kv.js'; +import kv, { runKv, askKv } from '../../../src/commands/kv.js'; import { runServe, resolveOptions } from '../../../src/commands/serve.js'; import { getCommand, getCommandOptions } from '../../../src/commandLoader.js'; @@ -51,6 +51,41 @@ describe('kv command', () => { expect(getCommandOptions()).to.deep.include({ value: 'kv', label: 'Read and write the local KV store' }); }); + describe('menu', () => { + const CANCEL = Symbol('cancel'); + function prompts(answers) { + const asked = []; + const next = (kind) => async (question) => { + asked.push({ kind, ...question }); + return answers.shift(); + }; + return { asked, prompts: { select: next('select'), text: next('text'), confirm: next('confirm'), isCancel: (value) => value === CANCEL } }; + } + + it('asks set --secret as a yes/no question, default no', async () => { + const fake = prompts(['set', 'orbit', 'api:token', '"x"', true]); + expect(await askKv(fake.prompts)).to.deep.equal({ local: true, action: 'set', store: 'orbit', key: 'api:token', value: '"x"', secret: true }); + const question = fake.asked.find((item) => item.kind === 'confirm'); + expect(question.initialValue).to.equal(false); + expect(question.message).to.match(/secret/i); + }); + + it('gives secret false when the answer is no', async () => { + const fake = prompts(['set', 'orbit', 'a', '1', false]); + expect((await askKv(fake.prompts)).secret).to.equal(false); + }); + + it('does not ask about a secret for get', async () => { + const fake = prompts(['get', 'orbit', 'a']); + expect(await askKv(fake.prompts)).to.deep.equal({ local: true, action: 'get', store: 'orbit', key: 'a', value: undefined }); + expect(fake.asked.some((item) => item.kind === 'confirm')).to.equal(false); + }); + + it('returns null when the secret question is cancelled', async () => { + expect(await askKv(prompts(['set', 'orbit', 'a', '1', CANCEL]).prompts)).to.equal(null); + }); + }); + it('refuses a run without --local', async () => { expect(await run(['get', 'orbit', 'a'])).to.equal(1); expect(err).to.deep.equal(['Remote KV commands are not available yet. Use --local.']); From 6bef29dea09dbdc40ef0cfe240ac3e556ff27510 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:52:44 +1000 Subject: [PATCH 07/32] fix(rules): keep the advice when a 409 or 422 body has only an error code A string error without a message is printed above the old advice (retry for 409, check URL and scopes for 422). A message still replaces the advice. deployError moves its text into helpers (complexity 10 or less). --- src/commands/rules.js | 38 +++++++++++++++++++----------- tests/unit/commands/rules.test.mjs | 12 ++++++++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/commands/rules.js b/src/commands/rules.js index 2b4f01d..9bb38cf 100644 --- a/src/commands/rules.js +++ b/src/commands/rules.js @@ -62,25 +62,35 @@ function refusalError(rules) { const MESSAGE_STATUSES = [409, 422]; const MAX_MESSAGE = 300; -// The `message` (else `error`) text of a 409 or 422 body, as one short line, or null. -function serverMessage(status, data) { - if (!MESSAGE_STATUSES.includes(status)) return null; - const text = [data?.message, data?.error].find(value => typeof value === 'string' && value.trim() !== ''); - if (text === undefined) return null; - const line = text.replace(/[\u0000-\u001f\u007f-\u009f]+/g, ' ').replace(/\s+/g, ' ').trim(); +// A string as one short line, or null when it is not text. +function oneLine(value) { + if (typeof value !== 'string' || value.trim() === '') return null; + const line = value.replace(/[\u0000-\u001f\u007f-\u009f]+/g, ' ').replace(/\s+/g, ' ').trim(); return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE - 3)}...` : line; } +function adviceText(status, code) { + if (status === 409) return 'Rules are being changed by another request (409). Retry shortly.'; + return `Rules deployment failed (${status || code || 'network error'}). Check portal URL, token scopes and project access.`; +} + +// A 409 or 422 `message` replaces the advice: the API writes it for the user +// (for example, that the change may already be live, or that the rule order +// cannot be kept). A string `error` alone is printed above the advice. +function statusText(status, data, code) { + const advice = adviceText(status, code); + if (!MESSAGE_STATUSES.includes(status)) return advice; + const message = oneLine(data?.message); + if (message) return `Rules deployment failed (${status}): ${message}`; + const errorText = oneLine(data?.error); + return errorText ? `Rules deployment failed (${status}): ${errorText}\n${advice}` : advice; +} + // Never print Axios config/headers or an arbitrary upstream body containing credentials. -// A 409 or 422 message is printed: the API writes it for the user (for example, -// that the change may already be live, or that the rule order cannot be kept). function deployError(error) { - const status = error.response?.status; - if (status === 409 && Array.isArray(error.response?.data?.rules)) return refusalError(error.response.data.rules); - const message = serverMessage(status, error.response?.data); - if (message) return new Error(`Rules deployment failed (${status}): ${message}`); - if (status === 409) return new Error('Rules are being changed by another request (409). Retry shortly.'); - return new Error(`Rules deployment failed (${status || error.code || 'network error'}). Check portal URL, token scopes and project access.`); + const { status, data } = error.response || {}; + if (status === 409 && Array.isArray(data?.rules)) return refusalError(data.rules); + return new Error(statusText(status, data, error.code)); } export default { diff --git a/tests/unit/commands/rules.test.mjs b/tests/unit/commands/rules.test.mjs index 59685ae..3e99910 100644 --- a/tests/unit/commands/rules.test.mjs +++ b/tests/unit/commands/rules.test.mjs @@ -123,8 +123,16 @@ describe('Rules deployment', () => { expect(message).to.equal(`Rules deployment failed (422): ${text}`); expect(message).not.to.match(/retry/i); }); - it('prints the error text of a 422 when the body has no message', async () => { - expect(await failure({ status: 422, data: { error: 'The rule order cannot be kept.' } })).to.equal('Rules deployment failed (422): The rule order cannot be kept.'); + it('prints the error text of a 422 and keeps the advice when the body has no message', async () => { + expect(await failure({ status: 422, data: { error: 'rule_order' } })).to.equal( + 'Rules deployment failed (422): rule_order\nRules deployment failed (422). Check portal URL, token scopes and project access.'); + }); + it('prints the error text of a 409 and keeps the retry advice when the body has no message', async () => { + expect(await failure({ status: 409, data: { error: 'conflict' } })).to.equal( + 'Rules deployment failed (409): conflict\nRules are being changed by another request (409). Retry shortly.'); + }); + it('uses the message, not the error text, when the body has both', async () => { + expect(await failure({ status: 409, data: { error: 'conflict', message: 'Reload and check.' } })).to.equal('Rules deployment failed (409): Reload and check.'); }); it('keeps the generic text for a 422 without a body', async () => { expect(await failure({ status: 422 })).to.equal('Rules deployment failed (422). Check portal URL, token scopes and project access.'); From 79db7fa9c9e3d3b852b06312a9b6e75182df13bc Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:52:44 +1000 Subject: [PATCH 08/32] fix(serve): stop the wait for the --proxy URL at once when quant serve stops stopRun aborts the poll, so it never keeps Node alive after a stop. --- src/commands/serve.js | 45 ++++++++++++++++++++++++------ tests/unit/commands/serve.test.mjs | 23 +++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/commands/serve.js b/src/commands/serve.js index a486e9e..a3f50b5 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -257,9 +257,11 @@ function startRun(command, root, print, proxy) { stdio: POSIX ? ['ignore', 'inherit', 'inherit'] : 'inherit' }); child.on('error', (err) => print(`The --run command could not start: ${err.message}`)); + // stopRun aborts the wait for the --proxy URL, so that it never keeps Node alive. + child.runAbort = new AbortController(); child.on('exit', (code, signal) => { if (child.stopping) return; - runExitLine(child, { code, signal, proxy }).then((line) => { + runExitLine(child, { code, signal, proxy, abort: child.runAbort.signal }).then((line) => { if (!child.stopping) print(line); }); }); @@ -276,23 +278,43 @@ function groupAlive(pid) { } } -async function answers(url) { +function requestSignal(abort) { + const timeout = AbortSignal.timeout(500); + return abort && AbortSignal.any ? AbortSignal.any([timeout, abort]) : timeout; +} + +async function answers(url, abort) { try { - await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(500) }); + await fetch(url, { redirect: 'manual', signal: requestSignal(abort) }); return true; } catch (_err) { return false; } } +// A delay that ends at once when abort fires. +function pause(ms, abort) { + return new Promise((resolve) => { + const timer = setTimeout(done, ms); + function done() { + clearTimeout(timer); + if (abort) abort.removeEventListener('abort', done); + resolve(); + } + if (abort) abort.addEventListener('abort', done, { once: true }); + }); +} + // True when the dev server answers within waitMs (it can still be starting). -async function reachable(url, waitMs) { +// False at once when abort fires. +async function reachable(url, waitMs, abort) { const deadline = Date.now() + waitMs; - for (;;) { - if (await answers(url)) return true; + while (!(abort && abort.aborted)) { + if (await answers(url, abort)) return true; if (Date.now() >= deadline) return false; - await new Promise((resolve) => setTimeout(resolve, 200)); + await pause(200, abort); } + return false; } /** @@ -302,13 +324,13 @@ async function reachable(url, waitMs) { * URL still answers (the dev server left the group, or on Windows), quant * serve cannot stop it. */ -export async function runExitLine(child, { code, signal, proxy, waitMs = RUN_BACKGROUND_WAIT_MS }) { +export async function runExitLine(child, { code, signal, proxy, waitMs = RUN_BACKGROUND_WAIT_MS, abort }) { const stopped = `The --run command stopped (${signal || `exit code ${code}`}).`; if (signal || code !== 0) return stopped; if (groupAlive(child.pid)) { return 'The --run command moved to the background (exit code 0). quant serve keeps serving and stops its process group when quant serve stops.'; } - if (proxy && await reachable(proxy, waitMs)) { + if (proxy && await reachable(proxy, waitMs, abort)) { return `The --run command exited (exit code 0), but the dev server at ${proxy} still answers. quant serve keeps serving, but it cannot stop that dev server. Stop it yourself.`; } return stopped; @@ -335,11 +357,16 @@ async function waitForGroupEnd(pid, graceMs) { return !groupAlive(pid); } +function abortExitWait(child) { + if (child && child.runAbort) child.runAbort.abort(); +} + /** * Stop the --run command: SIGTERM to its process group, then SIGKILL when * the group is still alive after the grace period. */ export async function stopRun(child, { graceMs = RUN_GRACE_MS } = {}) { + abortExitWait(child); if (!child || !child.pid || (!POSIX && !running(child))) return; child.stopping = true; signalGroup(child, 'SIGTERM'); diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 2ed5fb3..08b45f5 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -11,6 +11,7 @@ import serve, { runServe, startupLines, stopRun, + runExitLine, versionInfo, STOP_SIGNALS } from '../../../src/commands/serve.js'; @@ -447,6 +448,28 @@ describe('serve command', () => { } }); + it('stops the wait for the --proxy URL at once when it is aborted', async () => { + const port = await freePort(); + const abort = new AbortController(); + const started = Date.now(); + setTimeout(() => abort.abort(), 100); + const line = await runExitLine({ pid: 999999 }, { code: 0, signal: null, proxy: `http://127.0.0.1:${port}`, waitMs: 5000, abort: abort.signal }); + expect(Date.now() - started).to.be.below(1000); + expect(line).to.equal('The --run command stopped (exit code 0).'); + }); + + it('aborts the wait for the --proxy URL when quant serve stops', async function () { + if (process.platform === 'win32') this.skip(); + const port = await freePort(); + const served = serveWith(`${JSON.stringify(process.execPath)} -e "process.exit(0)"`, `http://127.0.0.1:${port}`); + await new Promise((resolve) => setTimeout(resolve, 400)); + const started = Date.now(); + await served.stop(); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(Date.now() - started).to.be.below(1500); + expect(served.lines.filter((line) => line.startsWith('The --run command'))).to.deep.equal([]); + }); + it('reports a failed command as before', async () => { const served = serveWith(fixture('fail.mjs'), undefined); await until(() => served.lines.some((line) => line.startsWith('The --run command')), 'the exit notice'); From 762322572f50cbdac0842675a3162b8246a55e10 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:52:45 +1000 Subject: [PATCH 09/32] fix(kv): never print a secret value after kv set --secret set prints the entry with the value [ENCRYPTED], the mask of a list. The menu asks for a secret value with hidden input. --secret on get, list or delete prints one notice that it is ignored. --- src/commands/kv.js | 23 +++++++++++++------- tests/unit/commands/kv.test.mjs | 37 +++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/commands/kv.js b/src/commands/kv.js index f7c4446..2e42a1f 100644 --- a/src/commands/kv.js +++ b/src/commands/kv.js @@ -6,7 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; -import { select, text, confirm, isCancel } from '@clack/prompts'; +import { select, text, password, confirm, isCancel } from '@clack/prompts'; import { openLocalKv } from '../serve/kv/wire.js'; import { pendingLedgerWarnings } from '../serve/kv/ledgers.js'; import { DEFAULT_FILES } from '../serve/project.js'; @@ -16,6 +16,8 @@ import { resolveOptions } from './serve.js'; const REMOTE_MESSAGE = 'Remote KV commands are not available yet. Use --local.'; const ACTIONS = ['list', 'get', 'set', 'delete']; const MAX_PAGES = 100; +// The mask of an encrypted value, the same as in a list. +const ENCRYPTED = '[ENCRYPTED]'; /** A failure with a message for the user and no stack. */ class UsageError extends Error {} @@ -106,7 +108,9 @@ const OPERATIONS = { async set(store, args) { const value = parseJson(need(args.value, 'The JSON value')); await store.set(need(args.key, 'The key'), value, setOptions(args)); - return store.getEntry(args.key); + const entry = await store.getEntry(args.key); + // Never print a secret value after a write: it would go to logs and scrollback. + return entry && entry.encrypted ? { ...entry, value: ENCRYPTED } : entry; }, async delete(store, args) { await store.delete(need(args.key, 'The key')); @@ -121,6 +125,7 @@ function describeError(err) { async function execute(args, root, error) { if (!ACTIONS.includes(args.action)) throw new UsageError(`The action must be one of: ${ACTIONS.join(', ')}`); + if (args.secret === true && args.action !== 'set') error(`--secret applies only to set. It is ignored for ${args.action}.`); const options = resolveOptions({ org: args.org, project: args.project, meta: args.meta }, { root, gitUserName: () => null }); const id = idOf(options, args.store); if (id === null) throw new UsageError(`FORBIDDEN: Access to KV store ${args.store} is not allowed`); @@ -153,19 +158,21 @@ export async function runKv(args, { root = process.cwd(), print = console.log, e } } -const PROMPTS = { select, text, confirm, isCancel }; +const PROMPTS = { select, text, password, confirm, isCancel }; -// set: the JSON value and the secret question (default no). Null on cancel. +// set: the secret question (default no), then the JSON value. A secret +// value is typed with hidden input. Null on cancel. async function askValue(prompts) { - const value = await prompts.text({ message: 'JSON value' }); - if (prompts.isCancel(value)) return null; const secret = await prompts.confirm({ message: 'Store the value as a secret (encrypted)?', initialValue: false }); - return prompts.isCancel(secret) ? null : { value, secret: secret === true }; + if (prompts.isCancel(secret)) return null; + const ask = secret === true ? prompts.password : prompts.text; + const value = await ask({ message: 'JSON value' }); + return prompts.isCancel(value) ? null : { value, secret: secret === true }; } /** * The menu: ask for the action, the store and what the action needs. - * @param {{ select, text, confirm, isCancel }} [prompts] The prompt functions (for tests). + * @param {{ select, text, password, confirm, isCancel }} [prompts] The prompt functions (for tests). * @returns {Promise} The args, or null on cancel. */ export async function askKv(prompts = PROMPTS) { diff --git a/tests/unit/commands/kv.test.mjs b/tests/unit/commands/kv.test.mjs index 42a464c..9d568e8 100644 --- a/tests/unit/commands/kv.test.mjs +++ b/tests/unit/commands/kv.test.mjs @@ -59,20 +59,25 @@ describe('kv command', () => { asked.push({ kind, ...question }); return answers.shift(); }; - return { asked, prompts: { select: next('select'), text: next('text'), confirm: next('confirm'), isCancel: (value) => value === CANCEL } }; + return { + asked, + prompts: { select: next('select'), text: next('text'), password: next('password'), confirm: next('confirm'), isCancel: (value) => value === CANCEL } + }; } - it('asks set --secret as a yes/no question, default no', async () => { - const fake = prompts(['set', 'orbit', 'api:token', '"x"', true]); + it('asks set --secret as a yes/no question, default no, then the value with hidden input', async () => { + const fake = prompts(['set', 'orbit', 'api:token', true, '"x"']); expect(await askKv(fake.prompts)).to.deep.equal({ local: true, action: 'set', store: 'orbit', key: 'api:token', value: '"x"', secret: true }); const question = fake.asked.find((item) => item.kind === 'confirm'); expect(question.initialValue).to.equal(false); expect(question.message).to.match(/secret/i); + expect(fake.asked[fake.asked.length - 1].kind).to.equal('password'); }); - it('gives secret false when the answer is no', async () => { - const fake = prompts(['set', 'orbit', 'a', '1', false]); - expect((await askKv(fake.prompts)).secret).to.equal(false); + it('gives secret false and a visible value prompt when the answer is no', async () => { + const fake = prompts(['set', 'orbit', 'a', false, '1']); + expect(await askKv(fake.prompts)).to.include({ secret: false, value: '1' }); + expect(fake.asked[fake.asked.length - 1].kind).to.equal('text'); }); it('does not ask about a secret for get', async () => { @@ -82,7 +87,8 @@ describe('kv command', () => { }); it('returns null when the secret question is cancelled', async () => { - expect(await askKv(prompts(['set', 'orbit', 'a', '1', CANCEL]).prompts)).to.equal(null); + expect(await askKv(prompts(['set', 'orbit', 'a', CANCEL]).prompts)).to.equal(null); + expect(await askKv(prompts(['set', 'orbit', 'a', true, CANCEL]).prompts)).to.equal(null); }); }); @@ -145,7 +151,10 @@ describe('kv command', () => { describe('set --secret', () => { it('stores the value encrypted: get reads it, list masks it, the state file does not hold it', async () => { expect(await run(['--local', 'set', 'orbit', 'api:token', '"s3cr3t-value"', '--secret'])).to.equal(0); - expect(lastJson()).to.include({ key: 'api:token', value: 's3cr3t-value', encrypted: true }); + expect(lastJson()).to.include({ key: 'api:token', value: '[ENCRYPTED]', encrypted: true }); + expect(out.join('\n')).not.to.include('s3cr3t-value'); + expect(await run(['--local', 'get', 'orbit', 'api:token'])).to.equal(0); + expect(lastJson()).to.include({ value: 's3cr3t-value', encrypted: true }); expect(await run(['--local', 'list', 'orbit', '--prefix', 'api:'])).to.equal(0); expect(lastJson()).to.have.length(1); expect(lastJson()[0]).to.include({ key: 'api:token', encrypted: true, value: '[ENCRYPTED]' }); @@ -162,7 +171,17 @@ describe('kv command', () => { await run(['--local', 'set', 'orbit', 'a', '1']); const { version } = lastJson(); expect(await run(['--local', 'set', 'orbit', 'a', '2', '--secret', '--if-version', version])).to.equal(0); - expect(lastJson()).to.include({ value: 2, encrypted: true }); + expect(lastJson()).to.include({ value: '[ENCRYPTED]', encrypted: true }); + }); + + it('prints one notice that --secret is ignored on get, list and delete', async () => { + await run(['--local', 'set', 'orbit', 'a', '1']); + for (const action of ['get', 'list', 'delete']) { + err.length = 0; + const argv = action === 'list' ? ['--local', action, 'orbit', '--secret'] : ['--local', action, 'orbit', 'a', '--secret']; + expect(await run(argv)).to.equal(0); + expect(err).to.deep.equal([`--secret applies only to set. It is ignored for ${action}.`]); + } }); it('refuses a secret in an indexed collection', async () => { From dcd0a950f0ed9864fcf7f796eb815926ba40885b Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:52:45 +1000 Subject: [PATCH 10/32] docs: secret output, quant/serve.json errors and store ids, --project, --run limits --- README.md | 8 +++++--- docs/adopting-quant-serve.md | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3b3f75e..1a3147b 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ Flags: - `--meta` - Template metadata with the KV stores (default: `meta` in `quant/serve.json`, then `quant/meta.json`). - `--fixtures` - Directory of KV fixtures (default: `quant/kv-fixtures`). - `--org` - Organisation (default: `org` in `quant/serve.json`, the `quant.json` clientid, then `local`). -- `--project` - Project (default: the `quant.json` project, then `dev`). +- `--project` - Project (default: `project` in `quant/serve.json`, the `quant.json` project, then `dev`). `quant serve` has no `--project` option of its own: it uses the global `--project` option of the CLI, so `quant serve --help` lists it with the global options. - `--domain` - Host for the domain match of rules (default: the Host of the request). - `--user-name` - Value for `{user_name}` in KV seed values (default: `git config user.name`, then `Owner`). - `--build` - Command to run once before the server starts. @@ -311,7 +311,7 @@ Flags: - `--index` - `list`: index name. - `--equals` - `list`: `field=value` for the index. Repeat the flag for more fields. - `--if-version` - `set`: write only if the key has this version. -- `--secret` - `set`: store the value encrypted, as `set(key, value, { secret: true })` in a function does. A function reads the value as usual. `list` shows `encrypted: true` and the value `[ENCRYPTED]`. `get --local` shows the value, because the local store is yours. A key in an indexed collection cannot hold a secret (`INDEX_FIELD_ENCRYPTED`). The local store uses local keys only: do not put production secrets in it. +- `--secret` - `set`: store the value encrypted, as `set(key, value, { secret: true })` in a function does. A function reads the value as usual. `list` shows `encrypted: true` and the value `[ENCRYPTED]`. `set` prints the entry with the value `[ENCRYPTED]`, so the value does not go into the terminal output or CI logs. `get --local` shows the value, because the local store is yours. The value is a command argument, so it goes into your shell history; the interactive menu asks for a secret value with hidden input. With `get`, `list` or `delete`, `--secret` is ignored and the command prints one notice. A key in an indexed collection cannot hold a secret (`INDEX_FIELD_ENCRYPTED`). The local store uses local keys only: do not put production secrets in it. - `--org` - Organisation (default: `org` in `quant/serve.json`, the `quant.json` clientid, then `local`). - `--project` - Project (default: `project` in `quant/serve.json`, the `quant.json` project, then `dev`). - `--meta` - Template metadata with the KV stores (default: `meta` in `quant/serve.json`, then `quant/meta.json`). @@ -455,13 +455,15 @@ Each field is optional. For `org`, `project` and `meta`, the order is: the flag, then `quant/serve.json`, then `quant.json` (org and project only), then the default (`local`, `dev`, `quant/meta.json`). `quant serve` reads these three fields only at start-up, also with `--watch`. If `quant/serve.json` and `quant.json` set a different org (or project) and no flag chooses, `quant serve` and `quant kv --local` print one warning line that names the value they use. +The local store ids contain the org and the project (for example `local-dev-orbit`). If you add or change `org` or `project` in `quant/serve.json`, the store ids change (for example to `acme-dev-orbit`), and the local data that you wrote before seems to be missing. The data is still in `.quant/kv/`. Set the old values again to see it. + ```json { "org": "acme", "project": "marketing-site" } ``` The order is the order of the edge: global headers, basic authentication, rules, the trailing-slash redirect, then static content. -`quant/serve.json` is local only. No deploy command reads it, and `quant deploy` never uploads it. `quant serve` never prints the password. Do not put a production password in the file. A file with the wrong shape stops `quant serve` at start-up, with a message that names the field. With `--watch`, `quant serve` reads the file again when it changes. If the new file has the wrong shape, `quant serve` prints the message and keeps the last valid settings. +`quant/serve.json` is local only. No deploy command reads it, and `quant deploy` never uploads it. `quant serve` never prints the password. Do not put a production password in the file. A file with the wrong shape stops `quant serve` at start-up, with a message that names the field. It also stops `quant kv --local` (exit code 1), because that command reads the `org`, `project` and `meta` defaults from the same file. With `--watch`, `quant serve` reads the file again when it changes. If the new file has the wrong shape, `quant serve` prints the message and keeps the last valid settings. ## Edge libraries in quant serve diff --git a/docs/adopting-quant-serve.md b/docs/adopting-quant-serve.md index 1f888c6..8a1d517 100644 --- a/docs/adopting-quant-serve.md +++ b/docs/adopting-quant-serve.md @@ -51,7 +51,10 @@ bundles function sources into the committed function bodies. In some environments the dev server moves to the background and the `--run` command exits 0 at once. quant serve then prints that the command moved to the - background, keeps serving, and stops the dev server when it stops. + background and keeps serving. On macOS and Linux, it stops the dev server when + it stops, but only while the dev server stays in the process group of the + `--run` command. On Windows, quant serve cannot stop a dev server that moved to + the background. See the `--run` flag in the README. 3. Move the seed data into fixtures. Each store gets one file, `quant/kv-fixtures/.json`, with a JSON array of entries: From 2d4fb87f57d6dffe162beef95df9306d0a560fe0 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 18:52:52 +1000 Subject: [PATCH 11/32] release: 7.3.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index e1a671e..5d7b0dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quantcdn/quant-cli", - "version": "7.2.0", + "version": "7.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@quantcdn/quant-cli", - "version": "7.2.0", + "version": "7.3.0", "license": "ISC", "dependencies": { "@clack/core": "^1.0.0", diff --git a/package.json b/package.json index 7628e0b..05b5d28 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@quantcdn/quant-cli", - "version": "7.2.0", + "version": "7.3.0", "description": "Deploy tools for QuantCDN", "type": "module", "main": "cli.js", From 14b3a4a6b061cc1397ea507a7b5b37f8b5bd4b3a Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 21:52:03 +1000 Subject: [PATCH 12/32] feat(serve): portal API client for read-only pulls --- src/commands/rules.js | 25 +--- src/serve/portal/client.js | 123 +++++++++++++++++++ tests/unit/serve/portal-client.test.mjs | 156 ++++++++++++++++++++++++ 3 files changed, 282 insertions(+), 22 deletions(-) create mode 100644 src/serve/portal/client.js create mode 100644 tests/unit/serve/portal-client.test.mjs diff --git a/src/commands/rules.js b/src/commands/rules.js index 9bb38cf..6213034 100644 --- a/src/commands/rules.js +++ b/src/commands/rules.js @@ -3,24 +3,13 @@ import axios from 'axios'; import { text, isCancel } from '@clack/prompts'; import { assertManifest } from '../helper/functions-manifest.js'; import { resolveRules } from '../helper/edge-rules-manifest.js'; +import { portalOptions, oneLine } from '../serve/portal/client.js'; export { resolveRules }; export function rulesOptions(args, env = process.env) { - let saved = {}; - try { saved = JSON.parse(fs.readFileSync('quant.json', 'utf8')); } catch { /* Optional project selection only. */ } - const customer = args.clientid || args.c || env.QUANT_CLIENT_ID || env.QUANT_CUSTOMER || saved.clientid; - const project = args.project || args.p || env.QUANT_PROJECT || saved.project; - // Deliberately never read QUANT_TOKEN, -t, saved.token, or upload endpoint. - const token = args['api-token'] || env.QUANT_API_TOKEN; - const base = args['api-base-url'] || env.QUANT_BASE_URL; - if (!customer || !project || !token || !base) throw new Error('Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'); - const url = new URL(base); - if (url.username || url.password || url.search || url.hash || !/\/api\/v2\/?$/.test(url.pathname) || - (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)))) { - throw new Error('Rules require an HTTPS portal /api/v2 URL (HTTP is allowed only for localhost)'); - } - return { customer, project, token, base: base.replace(/\/$/, '') }; + const { base, ...rest } = portalOptions(args, env); + return { ...rest, base: base.href.replace(/\/$/, '') }; } // Drift values printed in CI logs: matchers and placement only. Anything else @@ -60,14 +49,6 @@ function refusalError(rules) { } const MESSAGE_STATUSES = [409, 422]; -const MAX_MESSAGE = 300; - -// A string as one short line, or null when it is not text. -function oneLine(value) { - if (typeof value !== 'string' || value.trim() === '') return null; - const line = value.replace(/[\u0000-\u001f\u007f-\u009f]+/g, ' ').replace(/\s+/g, ' ').trim(); - return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE - 3)}...` : line; -} function adviceText(status, code) { if (status === 409) return 'Rules are being changed by another request (409). Retry shortly.'; diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js new file mode 100644 index 0000000..10b7e22 --- /dev/null +++ b/src/serve/portal/client.js @@ -0,0 +1,123 @@ +import fs from 'node:fs'; + +const REQUEST_TIMEOUT_MS = 15000; +const DEFAULT_RETRY_SECONDS = 2; +const MAX_WAIT_MS = 30000; +const MAX_MESSAGE = 300; +const LOCAL_HOSTS = ['localhost', '127.0.0.1', '[::1]']; + +export class PortalError extends Error { + constructor(message, status) { + super(message); + this.name = 'PortalError'; + this.status = status; + } +} + +// A string as one short line, or null when it is not text. +export function oneLine(value) { + if (typeof value !== 'string' || value.trim() === '') return null; + const line = value.replace(/[\u0000-\u001f\u007f-\u009f]+/g, ' ').replace(/\s+/g, ' ').trim(); + return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE - 3)}...` : line; +} + +function savedProject() { + try { + return JSON.parse(fs.readFileSync('quant.json', 'utf8')); + } catch { + return {}; // Optional project selection only. + } +} + +function assertPortalUrl(url) { + const plainLocal = url.protocol === 'http:' && LOCAL_HOSTS.includes(url.hostname); + if (url.username || url.password || url.search || url.hash || !/\/api\/v2\/?$/.test(url.pathname) || + (url.protocol !== 'https:' && !plainLocal)) { + throw new Error('Rules require an HTTPS portal /api/v2 URL (HTTP is allowed only for localhost)'); + } +} + +const first = (...values) => values.find(Boolean); + +export function portalOptions(args, env = process.env) { + const saved = savedProject(); + const customer = first(args.clientid, args.c, env.QUANT_CLIENT_ID, env.QUANT_CUSTOMER, saved.clientid); + const project = first(args.project, args.p, env.QUANT_PROJECT, saved.project); + // Deliberately never read QUANT_TOKEN, -t, saved.token, or upload endpoint. + const token = first(args['api-token'], env.QUANT_API_TOKEN); + const base = first(args['api-base-url'], env.QUANT_BASE_URL); + if (![customer, project, token, base].every(Boolean)) throw new Error('Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'); + const url = new URL(base); + assertPortalUrl(url); + return { customer, project, token, base: url }; +} + +function buildUrl(base, path, query) { + const url = new URL(`${String(base.href).replace(/\/$/, '')}${path}`); + for (const [key, value] of Object.entries(query || {})) url.searchParams.set(key, String(value)); + return url; +} + +function retryDelayMs(response) { + const header = response.headers.get('retry-after'); + const seconds = header === null || header.trim() === '' ? NaN : Number(header); + return (Number.isFinite(seconds) && seconds >= 0 ? seconds : DEFAULT_RETRY_SECONDS) * 1000; +} + +async function readBody(response) { + try { + return await response.json(); + } catch { + return null; + } +} + +function failureMessage(status, body, token) { + const message = oneLine(body?.message)?.split(token).join('[redacted]'); + return message ? `The portal answered ${status}: ${message}` : `The portal answered ${status}.`; +} + +function refusal(status, scopeInfo) { + if (status === 401) return new PortalError('The portal refused QUANT_API_TOKEN (401).', 401); + const { scope = 'required', what = 'this request' } = scopeInfo || {}; + return new PortalError(`QUANT_API_TOKEN lacks the ${scope} scope for ${what}.`, 403); +} + +async function send({ base, token, fetchImpl }, path, query) { + try { + return await fetchImpl(buildUrl(base, path, query), { + method: 'GET', + headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' }, + redirect: 'manual', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }); + } catch { + // The original error is dropped on purpose: it can echo request headers. + throw new PortalError('The portal did not answer. Check QUANT_BASE_URL and the network.'); + } +} + +async function settle(context, response, scopeInfo) { + const { status } = response; + if (status === 401 || status === 403) throw refusal(status, scopeInfo); + if (status === 404) return { status }; + const body = await readBody(response); + if (status < 200 || status > 299) throw new PortalError(failureMessage(status, body, context.token), status); + return { status, body }; +} + +export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) }) { + const context = { base, token, fetchImpl }; + async function get(path, query, scopeInfo) { + let waited = 0; + for (;;) { + const response = await send(context, path, query); + if (response.status !== 429) return settle(context, response, scopeInfo); + const delay = retryDelayMs(response); + if (waited + delay > MAX_WAIT_MS) throw new PortalError('The portal rate limit (429) did not clear in 30 seconds. Try again later.', 429); + waited += delay; + await sleep(delay); + } + } + return { get }; +} diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs new file mode 100644 index 0000000..3c13a05 --- /dev/null +++ b/tests/unit/serve/portal-client.test.mjs @@ -0,0 +1,156 @@ +import { inspect } from 'node:util'; +import { expect } from 'chai'; +import { portalOptions, createPortalClient, PortalError } from '../../../src/serve/portal/client.js'; + +const TOKEN = 'test-token'; +const BASE = new URL('http://127.0.0.1:8080/api/v2'); + +const reply = (status, body, headers = {}) => ({ + status, + headers: { get: name => headers[name.toLowerCase()] ?? null }, + json: async () => { + if (body === undefined) throw new Error('no body'); + return body; + } +}); + +function setup(responses) { + const calls = []; + const waits = []; + const queue = [...responses]; + const fetchImpl = async (url, init) => { + calls.push({ url: String(url), init }); + const next = queue.shift(); + if (next instanceof Error) throw next; + return next; + }; + const sleep = async ms => { waits.push(ms); }; + return { calls, waits, client: createPortalClient({ base: BASE, token: TOKEN, fetchImpl, sleep }) }; +} + +async function failure(promise) { + try { + await promise; + } catch (error) { + return error; + } + throw new Error('expected a rejection'); +} + +describe('portal client', () => { + describe('portalOptions', () => { + const args = { c: 'org', p: 'site' }; + const env = { QUANT_API_TOKEN: TOKEN, QUANT_BASE_URL: 'http://127.0.0.1:8080/api/v2/' }; + + it('returns the base as a URL with the token, customer and project', () => { + const options = portalOptions(args, env); + expect(options.base).to.be.instanceOf(URL); + expect(options.base.hostname).to.equal('127.0.0.1'); + expect(options).to.include({ token: TOKEN, customer: 'org', project: 'site' }); + }); + + it('asks for the missing values', () => { + expect(() => portalOptions(args, {})).to.throw('QUANT_API_TOKEN'); + }); + + it('refuses plain HTTP to a remote host', () => { + expect(() => portalOptions(args, { ...env, QUANT_BASE_URL: 'http://example.com/api/v2' })).to.throw('HTTPS'); + }); + }); + + describe('get', () => { + it('sends the bearer token and JSON accept header without following redirects', async () => { + const { client, calls } = setup([reply(200, { ok: true })]); + const result = await client.get('/organizations/org/rules', { page: 2 }); + expect(result).to.deep.equal({ status: 200, body: { ok: true } }); + expect(calls[0].url).to.equal('http://127.0.0.1:8080/api/v2/organizations/org/rules?page=2'); + expect(calls[0].init.headers.Authorization).to.equal(`Bearer ${TOKEN}`); + expect(calls[0].init.headers.Accept).to.equal('application/json'); + expect(calls[0].init.redirect).to.equal('manual'); + expect(calls[0].init.signal).to.be.instanceOf(AbortSignal); + }); + + it('waits for Retry-After on 429 and then retries', async () => { + const { client, waits, calls } = setup([reply(429, {}, { 'retry-after': '3' }), reply(200, { ok: 1 })]); + const result = await client.get('/x'); + expect(result.status).to.equal(200); + expect(waits).to.deep.equal([3000]); + expect(calls).to.have.length(2); + }); + + it('waits 2 seconds on 429 without Retry-After', async () => { + const { client, waits } = setup([reply(429, {}), reply(200, {})]); + await client.get('/x'); + expect(waits).to.deep.equal([2000]); + }); + + it('throws a 429 error when the waits pass 30 seconds in total', async () => { + const { client, waits } = setup([ + reply(429, {}, { 'retry-after': '20' }), reply(429, {}, { 'retry-after': '20' }) + ]); + const error = await failure(client.get('/x')); + expect(error).to.be.instanceOf(PortalError); + expect(error.status).to.equal(429); + expect(waits).to.deep.equal([20000]); + }); + + it('stops on 401 and names the token', async () => { + const { client } = setup([reply(401, { message: 'nope' })]); + const error = await failure(client.get('/x')); + expect(error.status).to.equal(401); + expect(error.message).to.equal('The portal refused QUANT_API_TOKEN (401).'); + }); + + it('stops on 403 and names the scope', async () => { + const { client } = setup([reply(403, {})]); + const error = await failure(client.get('/x', undefined, { scope: 'rules:read', what: 'the rule list' })); + expect(error.status).to.equal(403); + expect(error.message).to.equal('QUANT_API_TOKEN lacks the rules:read scope for the rule list.'); + }); + + it('returns the status for 404', async () => { + const { client } = setup([reply(404, { message: 'gone' })]); + expect(await client.get('/x')).to.deep.equal({ status: 404 }); + }); + + it('throws the status and a cleaned server message for other errors', async () => { + const long = `bad\u0000\n${'x'.repeat(400)}`; + const { client } = setup([reply(500, { message: long })]); + const error = await failure(client.get('/x')); + expect(error).to.be.instanceOf(PortalError); + expect(error.status).to.equal(500); + expect(error.message).to.match(/: bad x+\.\.\.$/); + expect(error.message.length).to.be.at.most(330); + expect(error.message).to.not.match(/[\u0000-\u001f]/); + }); + + it('throws with the status alone when the body has no message', async () => { + const { client } = setup([reply(502, undefined)]); + const error = await failure(client.get('/x')); + expect(error.status).to.equal(502); + expect(error.message).to.include('502'); + }); + + it('turns a network failure into a PortalError without status', async () => { + const { client } = setup([new Error(`connect failed with ${TOKEN}`)]); + const error = await failure(client.get('/x')); + expect(error).to.be.instanceOf(PortalError); + expect(error.status).to.equal(undefined); + }); + + it('never puts the token in an error message or its inspected form', async () => { + const cases = [ + [reply(401, { message: TOKEN })], + [reply(403, { message: TOKEN })], + [reply(500, { message: `echo ${TOKEN}` })], + [new Error(`connect failed with ${TOKEN}`)] + ]; + for (const responses of cases) { + const { client } = setup(responses); + const error = await failure(client.get('/x')); + expect(error.message).to.not.include(TOKEN); + expect(inspect(error, { depth: 6 })).to.not.include(TOKEN); + } + }); + }); +}); From 00789eea2960ce63dd0f8e43c0e44128c3321e38 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 21:54:32 +1000 Subject: [PATCH 13/32] fix(serve): portal client review fixes --- src/commands/rules.js | 10 +++- src/serve/portal/client.js | 17 ++++--- tests/unit/serve/portal-client.test.mjs | 66 +++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/commands/rules.js b/src/commands/rules.js index 6213034..09898ca 100644 --- a/src/commands/rules.js +++ b/src/commands/rules.js @@ -8,8 +8,14 @@ import { portalOptions, oneLine } from '../serve/portal/client.js'; export { resolveRules }; export function rulesOptions(args, env = process.env) { - const { base, ...rest } = portalOptions(args, env); - return { ...rest, base: base.href.replace(/\/$/, '') }; + // Rules keep the token and base flags that the serve command does not have. + const merged = { + ...env, + QUANT_API_TOKEN: args['api-token'] || env.QUANT_API_TOKEN, + QUANT_BASE_URL: args['api-base-url'] || env.QUANT_BASE_URL + }; + const { base: _url, ...rest } = portalOptions(args, merged); + return { ...rest, base: merged.QUANT_BASE_URL.replace(/\/$/, '') }; } // Drift values printed in CI logs: matchers and placement only. Anything else diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js index 10b7e22..b2df90a 100644 --- a/src/serve/portal/client.js +++ b/src/serve/portal/client.js @@ -3,6 +3,7 @@ import fs from 'node:fs'; const REQUEST_TIMEOUT_MS = 15000; const DEFAULT_RETRY_SECONDS = 2; const MAX_WAIT_MS = 30000; +const MIN_RETRY_MS = 250; const MAX_MESSAGE = 300; const LOCAL_HOSTS = ['localhost', '127.0.0.1', '[::1]']; @@ -44,8 +45,8 @@ export function portalOptions(args, env = process.env) { const customer = first(args.clientid, args.c, env.QUANT_CLIENT_ID, env.QUANT_CUSTOMER, saved.clientid); const project = first(args.project, args.p, env.QUANT_PROJECT, saved.project); // Deliberately never read QUANT_TOKEN, -t, saved.token, or upload endpoint. - const token = first(args['api-token'], env.QUANT_API_TOKEN); - const base = first(args['api-base-url'], env.QUANT_BASE_URL); + // The serve command has no token flag: only the environment is read. + const { QUANT_API_TOKEN: token, QUANT_BASE_URL: base } = env; if (![customer, project, token, base].every(Boolean)) throw new Error('Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'); const url = new URL(base); assertPortalUrl(url); @@ -60,20 +61,21 @@ function buildUrl(base, path, query) { function retryDelayMs(response) { const header = response.headers.get('retry-after'); - const seconds = header === null || header.trim() === '' ? NaN : Number(header); - return (Number.isFinite(seconds) && seconds >= 0 ? seconds : DEFAULT_RETRY_SECONDS) * 1000; + const seconds = /^\d+$/.test(header ?? '') ? Number(header) : DEFAULT_RETRY_SECONDS; + return Math.max(seconds * 1000, MIN_RETRY_MS); } async function readBody(response) { try { return await response.json(); } catch { - return null; + return undefined; } } function failureMessage(status, body, token) { - const message = oneLine(body?.message)?.split(token).join('[redacted]'); + const raw = typeof body?.message === 'string' ? body.message.split(token).join('[redacted]') : null; + const message = oneLine(raw); return message ? `The portal answered ${status}: ${message}` : `The portal answered ${status}.`; } @@ -103,7 +105,8 @@ async function settle(context, response, scopeInfo) { if (status === 404) return { status }; const body = await readBody(response); if (status < 200 || status > 299) throw new PortalError(failureMessage(status, body, context.token), status); - return { status, body }; + if (body === undefined && status !== 204) throw new PortalError('The portal sent a response that is not JSON.', status); + return { status, body: body ?? null }; } export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) }) { diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index 3c13a05..c40f3dd 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -1,5 +1,6 @@ import { inspect } from 'node:util'; import { expect } from 'chai'; +import { rulesOptions } from '../../../src/commands/rules.js'; import { portalOptions, createPortalClient, PortalError } from '../../../src/serve/portal/client.js'; const TOKEN = 'test-token'; @@ -49,6 +50,19 @@ describe('portal client', () => { expect(options).to.include({ token: TOKEN, customer: 'org', project: 'site' }); }); + it('ignores the token and base flags', () => { + const flags = { ...args, 'api-token': 'flag-token', 'api-base-url': 'https://other.test/api/v2' }; + const options = portalOptions(flags, env); + expect(options.token).to.equal(TOKEN); + expect(options.base.hostname).to.equal('127.0.0.1'); + expect(() => portalOptions(flags, {})).to.throw('QUANT_API_TOKEN'); + }); + + it('keeps the flags and the old base string in rulesOptions', () => { + const flags = { ...args, 'api-token': 'flag-token', 'api-base-url': 'https://example.test/api/v2/' }; + expect(rulesOptions(flags, {})).to.deep.equal({ customer: 'org', project: 'site', token: 'flag-token', base: 'https://example.test/api/v2' }); + }); + it('asks for the missing values', () => { expect(() => portalOptions(args, {})).to.throw('QUANT_API_TOKEN'); }); @@ -94,6 +108,58 @@ describe('portal client', () => { expect(waits).to.deep.equal([20000]); }); + it('stops after a bounded number of 429 answers with Retry-After 0', async () => { + const { client, calls } = setup(Array.from({ length: 500 }, () => reply(429, {}, { 'retry-after': '0' }))); + const error = await failure(client.get('/x')); + expect(error.status).to.equal(429); + expect(calls.length).to.be.at.most(125); + }); + + it('uses the default delay for a Retry-After that is not whole seconds', async () => { + for (const value of ['Wed, 21 Oct 2026 07:28:00 GMT', '1e1', '0x10', '-1']) { + const { client, waits } = setup([reply(429, {}, { 'retry-after': value }), reply(200, {})]); + await client.get('/x'); + expect(waits).to.deep.equal([2000]); + } + }); + + it('throws when a 2xx body is not JSON', async () => { + const { client } = setup([reply(200, undefined)]); + const error = await failure(client.get('/x')); + expect(error).to.be.instanceOf(PortalError); + expect(error.status).to.equal(200); + expect(error.message).to.equal('The portal sent a response that is not JSON.'); + }); + + it('accepts a 204 without a body', async () => { + const { client } = setup([reply(204, undefined)]); + expect(await client.get('/x')).to.deep.equal({ status: 204, body: null }); + }); + + it('redacts the token before it cuts the message', async () => { + const message = `${'a'.repeat(280)}${TOKEN}${'b'.repeat(50)}`; + const { client } = setup([reply(500, { message })]); + const error = await failure(client.get('/x')); + expect(error.message).to.include('[redacted]'); + expect(error.message).to.not.include('test-tok'); + }); + + it('does not follow a redirect', async () => { + const { client, calls } = setup([reply(302, {}, { location: 'http://127.0.0.1:9/' })]); + const error = await failure(client.get('/x')); + expect(error.status).to.equal(302); + expect(calls).to.have.length(1); + expect(calls[0].init.redirect).to.equal('manual'); + }); + + it('turns a timeout into a PortalError without the token', async () => { + const abort = Object.assign(new Error(`aborted ${TOKEN}`), { name: 'TimeoutError' }); + const { client } = setup([abort]); + const error = await failure(client.get('/x')); + expect(error).to.be.instanceOf(PortalError); + expect(error.message).to.not.include(TOKEN); + }); + it('stops on 401 and names the token', async () => { const { client } = setup([reply(401, { message: 'nope' })]); const error = await failure(client.get('/x')); From a42d403864b18738ef086e850bb66dfc24a2afaf Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 21:56:06 +1000 Subject: [PATCH 14/32] test(serve): pin token redaction before truncation --- src/commands/rules.js | 4 ++-- tests/unit/serve/portal-client.test.mjs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/commands/rules.js b/src/commands/rules.js index 09898ca..88e182b 100644 --- a/src/commands/rules.js +++ b/src/commands/rules.js @@ -14,8 +14,8 @@ export function rulesOptions(args, env = process.env) { QUANT_API_TOKEN: args['api-token'] || env.QUANT_API_TOKEN, QUANT_BASE_URL: args['api-base-url'] || env.QUANT_BASE_URL }; - const { base: _url, ...rest } = portalOptions(args, merged); - return { ...rest, base: merged.QUANT_BASE_URL.replace(/\/$/, '') }; + const { customer, project, token } = portalOptions(args, merged); + return { customer, project, token, base: merged.QUANT_BASE_URL.replace(/\/$/, '') }; } // Drift values printed in CI logs: matchers and placement only. Anything else diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index c40f3dd..0685b69 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -137,10 +137,13 @@ describe('portal client', () => { }); it('redacts the token before it cuts the message', async () => { - const message = `${'a'.repeat(280)}${TOKEN}${'b'.repeat(50)}`; + const message = `${'a'.repeat(292)}${TOKEN}${'b'.repeat(50)}`; const { client } = setup([reply(500, { message })]); const error = await failure(client.get('/x')); - expect(error.message).to.include('[redacted]'); + expect(error.message).to.include('[red'); + for (let i = 0; i + 4 <= TOKEN.length; i++) { + for (let j = i + 4; j <= TOKEN.length; j++) expect(error.message).to.not.include(TOKEN.slice(i, j)); + } expect(error.message).to.not.include('test-tok'); }); From 88901b6eea677141ad516c82f0a244c99eda8e49 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:06:30 +1000 Subject: [PATCH 15/32] feat(serve): redirect, headers, custom response, basic auth, error page and proxy rules --- src/helper/edge-rules-manifest.js | 11 +- src/serve/forward.js | 35 ++ src/serve/project.js | 17 +- src/serve/rule-actions.js | 252 ++++++++++++ src/serve/rules.js | 48 ++- src/serve/server.js | 118 +++--- .../unit/helper/edge-rules-manifest.test.mjs | 7 + tests/unit/serve/rule-actions.test.mjs | 380 ++++++++++++++++++ tests/unit/serve/rules.test.mjs | 39 +- 9 files changed, 831 insertions(+), 76 deletions(-) create mode 100644 src/serve/forward.js create mode 100644 src/serve/rule-actions.js create mode 100644 tests/unit/serve/rule-actions.test.mjs diff --git a/src/helper/edge-rules-manifest.js b/src/helper/edge-rules-manifest.js index a0955b1..0bc16ef 100644 --- a/src/helper/edge-rules-manifest.js +++ b/src/helper/edge-rules-manifest.js @@ -87,11 +87,20 @@ export function ruleErrors(manifest) { })]; } -/** The rule as the local runtime matches it. `rule` is resolved (function_uuid set). */ +// The runtime action of each manifest rule type. +const RUNTIME_ACTION = { function: 'edge_fn', auth: 'edge_auth', filter: 'edge_filter' }; + +/** + * The rule as the local runtime matches it. `rule` is resolved (function_uuid set). + * `action` names what the rule does; `config` holds its action settings. + */ export function toRuntimeRule(rule) { return { id: rule.id, + name: rule.name ?? null, type: rule.type, + action: RUNTIME_ACTION[rule.type], + config: {}, url: rule.urls, domain: rule.domains?.length ? rule.domains.map(d => d.toLowerCase()) : ['any'], methods: rule.methods?.length ? rule.methods : null, diff --git a/src/serve/forward.js b/src/serve/forward.js new file mode 100644 index 0000000..57b0d57 --- /dev/null +++ b/src/serve/forward.js @@ -0,0 +1,35 @@ +/** + * The request and response headers and the request body of a forwarded + * request. The --proxy dev server stage and the proxy rule action use them. + */ + +const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'expect']; +const PROXY_REQUEST_DROP = [...HOP_BY_HOP, 'host', 'content-length']; +// Node fetch decodes the body, so the encoding and length headers are dropped. +export const PROXY_RESPONSE_DROP = [...HOP_BY_HOP, 'content-encoding', 'content-length']; + +export function withoutHeaders(source, names) { + const headers = new Headers(source); + for (const name of names) headers.delete(name); + return headers; +} + +/** The client request headers without the hop-by-hop, host and length headers. */ +export function proxyRequestHeaders(req) { + const headers = new Headers(); + for (let i = 0; i < req.rawHeaders.length; i += 2) headers.append(req.rawHeaders[i], req.rawHeaders[i + 1]); + return withoutHeaders(headers, PROXY_REQUEST_DROP); +} + +// The proxy streams the request body to the origin, unless a function +// (an auth rule that continued) has read it already. Then the proxy sends the +// capped body that the function saw. +export async function proxyBody(ctx) { + if (['GET', 'HEAD'].includes(ctx.method)) return {}; + if (ctx.bodyPending) { + const bytes = await ctx.bodyPending; + return bytes ? { body: bytes } : {}; + } + ctx.bodyStreamed = true; + return { body: ctx.req, duplex: 'half' }; +} diff --git a/src/serve/project.js b/src/serve/project.js index 9209988..2a17cd7 100644 --- a/src/serve/project.js +++ b/src/serve/project.js @@ -12,6 +12,7 @@ import { resolveRules, ruleErrors, toRuntimeRule } from '../helper/edge-rules-ma import { parseMeta } from '../helper/quant-meta.js'; import { readInlineSetting } from './inline.js'; import { readServeConfig, SERVE_CONFIG } from './project-config.js'; +import { actionWarnings } from './rule-actions.js'; import { orderRules } from './rules.js'; export const DEFAULT_FILES = Object.freeze({ @@ -76,9 +77,12 @@ function functionMap(root, manifest) { }])); } +const FUNCTION_ACTIONS = new Set(['edge_fn', 'edge_auth', 'edge_filter']); + function ruleWarning(rule, fn, source) { + if (!FUNCTION_ACTIONS.has(rule.action)) return null; if (!fn) { - const effect = rule.type === 'filter' ? ' The filter is skipped.' : ''; + const effect = rule.action === 'edge_filter' ? ' The filter is skipped.' : ''; return `Rule ${rule.id}: function ${rule.fnUuid} is not in ${source}.${effect}`; } return fs.existsSync(fn.path) ? null : `Rule ${rule.id}: the function file ${fn.path} was not found.`; @@ -88,17 +92,22 @@ function ruleWarning(rule, fn, source) { * Load and check the project files. A missing file counts as empty. * @param {Object} options * @param {string} options.root The project directory. File paths resolve from it. + * @param {Object[]} [options.runtimeRules] Runtime rules (with action and config) that run with + * the rules of the rules manifest, ordered by weight. * @returns {{ functions: Map, rules: Object[], stores: Object[], fixtures: string, inlineFunctions: boolean, warnings: string[] }} * functions is keyed by lowercase uuid; rules are runtime rules in run order. * inlineFunctions: project_settings.enable_inline_fn in meta.json. * settings: the local project settings from serve.json (see project-config.js). */ -export function loadProject({ root, functions = DEFAULT_FILES.functions, rules = DEFAULT_FILES.rules, meta = DEFAULT_FILES.meta, fixtures = DEFAULT_FILES.fixtures, serve = DEFAULT_FILES.serve }) { +export function loadProject({ root, functions = DEFAULT_FILES.functions, rules = DEFAULT_FILES.rules, meta = DEFAULT_FILES.meta, fixtures = DEFAULT_FILES.fixtures, serve = DEFAULT_FILES.serve, runtimeRules: extraRules = [] }) { const manifest = readFunctions(root, functions); - const runtimeRules = readRules(root, rules, manifest); + const runtimeRules = orderRules([...readRules(root, rules, manifest), ...extraRules]); const map = functionMap(root, manifest); const settings = readServeConfig(root, serve); - const warnings = runtimeRules.map((rule) => ruleWarning(rule, map.get(rule.fnUuid), functions)).filter(Boolean); + const warnings = [ + ...runtimeRules.map((rule) => ruleWarning(rule, map.get(rule.fnUuid), functions)).filter(Boolean), + ...actionWarnings(runtimeRules) + ]; return { functions: map, rules: runtimeRules, diff --git a/src/serve/rule-actions.js b/src/serve/rule-actions.js new file mode 100644 index 0000000..109feb7 --- /dev/null +++ b/src/serve/rule-actions.js @@ -0,0 +1,252 @@ +/** + * The rule actions of the `quant serve` rule stage. Each runtime rule names + * its `action` and holds its action settings in `config` (the rule's + * action_config). A rule's config can hold credentials (a basic-auth password, + * origin credentials, injected headers), so no message prints a config value: + * messages name the rule and its action only. + */ + +import { basicAuthResponse } from './project-config.js'; +import { PROXY_RESPONSE_DROP, proxyBody, proxyRequestHeaders, withoutHeaders } from './forward.js'; +import { ruleCaptures } from './rules.js'; + +const SUPPORTED = new Set(['edge_fn', 'edge_auth', 'edge_filter', 'redirect', 'headers', 'custom_response', 'auth', 'error_page', 'proxy']); +// Headers that a headers rule cannot set. +const PROTECTED = new Set(['quant-server', 'age', 'quant-project', 'quant-url-hash', 'quant-orig-host']); +const REDIRECT_CODES = [301, 302, 303, 307, 308]; +const NULL_BODY = [204, 205, 304]; +const DEFAULT_ORIGIN_TIMEOUT_MS = 30000; +const HTML = 'text/html; charset=utf-8'; +const PROXY_ERROR = 'Proxy Error' + + '

Proxy Error

An error occurred when communicating with a backend origin service.

'; + +/** The rule as messages name it: its name, else its id. */ +export const ruleLabel = (rule) => rule.name || rule.id; + +const configOf = (rule) => (rule.config && typeof rule.config === 'object' ? rule.config : {}); + +function validHeader(name, value) { + try { + new Headers([[name, value]]); + return true; + } catch (_err) { + return false; + } +} + +// The entries of a header map that are valid headers. An invalid entry is +// skipped, so that no error message carries its value. +function headerEntries(map) { + if (!map || typeof map !== 'object') return []; + return Object.entries(map) + .map(([name, value]) => [String(name).toLowerCase(), String(value)]) + .filter(([name, value]) => validHeader(name, value)); +} + +// A list of header names, without the names that are not valid. +function nameList(value) { + return Array.isArray(value) ? value.map(String).filter((name) => validHeader(name, '')) : []; +} + +function basicCredentials(user, pass) { + return `Basic ${Buffer.from(`${user ?? ''}:${pass ?? ''}`).toString('base64')}`; +} + +function failed(state, rule, why, status = 500) { + state.log(`Rule ${ruleLabel(rule)} (${rule.action}): ${why}.`); + return new Response(null, { status }); +} + +/** + * The destination as written, with $* (the whole path) and $1 to $5 (the + * text of each * of the pattern). A $n with no * stays as literal text. The + * query string is added unless the pattern contains ?. + */ +export function destination(template, ctx, captures) { + const groups = captures ? captures.groups : []; + const out = String(template ?? '').replace(/\$(\*|[1-5])/g, (match, key) => (key === '*' ? ctx.pathname : groups[key - 1] ?? match)); + const query = ctx.search || ''; + if ((captures && captures.withQuery) || query.length < 2) return out; + return `${out}${out.includes('?') ? '&' : '?'}${query.slice(1)}`; +} + +// --- Redirect, headers, custom response, basic auth ------------------------------------- + +function redirect(state, ctx, rule) { + const config = configOf(rule); + const location = destination(config.to, ctx, ruleCaptures(rule, ctx)); + if (!validHeader('location', location)) return failed(state, rule, 'the destination is not a valid Location header'); + const code = Number(config.status_code); + return new Response(null, { status: REDIRECT_CODES.includes(code) ? code : 301, headers: { location } }); +} + +function headers(_state, _ctx, rule) { + const entries = headerEntries(configOf(rule).headers).filter(([name]) => !PROTECTED.has(name)); + return { continue: true, headers: entries }; +} + +function customResponse(_state, _ctx, rule) { + const config = configOf(rule); + const code = Number(config.custom_response_status_code); + const status = Number.isInteger(code) && code >= 200 && code <= 599 ? code : 200; + const body = NULL_BODY.includes(status) ? null : String(config.custom_response_body ?? ''); + return new Response(body, { status, headers: { 'content-type': HTML } }); +} + +// A 401 when the credentials are missing or wrong; null (continue) when they are right. +function basicAuth(_state, ctx, rule) { + const config = configOf(rule); + const expected = basicCredentials(config.auth_user, config.auth_pass); + return basicAuthResponse({ basicAuth: expected, globalHeaders: [] }, ctx.req.headers.authorization); +} + +// --- Proxy ------------------------------------------------------------------------- + +// The origin of `to` up to its first placeholder. Only http and https. +function configuredOrigin(to) { + try { + const url = new URL(String(to ?? '').split('$')[0]); + return ['http:', 'https:'].includes(url.protocol) ? url.origin : null; + } catch (_err) { + return null; + } +} + +// The URL to forward to, or null when it is not on the configured origin +// (a placeholder value cannot move the request to another host). +function proxyTarget(rule, ctx) { + const config = configOf(rule); + const origin = configuredOrigin(config.to); + if (!origin) return null; + try { + const target = new URL(destination(config.to, ctx, ruleCaptures(rule, ctx))); + return target.origin === origin ? target : null; + } catch (_err) { + return null; + } +} + +// fetch sends the host of the target URL as Host, so the rule's host +// setting is not used (start-up warns). +function originRequestHeaders(ctx, config) { + const headers = withoutHeaders(proxyRequestHeaders(ctx.req), nameList(config.proxy_strip_request_headers)); + for (const [name, value] of headerEntries(config.inject_headers)) headers.set(name, value); + if (config.auth_user) headers.set('authorization', basicCredentials(config.auth_user, config.auth_pass)); + return headers; +} + +function originTimeout(config) { + const ms = Number(config.origin_timeout); + return Number.isFinite(ms) && ms > 0 ? ms : DEFAULT_ORIGIN_TIMEOUT_MS; +} + +// Empty or -1 keeps the origin's cache-control; 0 is no-store; N is public, max-age=N. +function applyCacheLifetime(target, value) { + const text = String(value ?? '').trim(); + const seconds = Number(text); + if (text === '' || !Number.isInteger(seconds) || seconds < 0) return; + target.set('cache-control', seconds === 0 ? 'no-store' : `public, max-age=${seconds}`); +} + +// The origin must answer within origin_timeout; the body then streams without a limit. +async function originFetch(ctx, config, target) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), originTimeout(config)); + try { + return await fetch(target, { + method: ctx.method, + headers: originRequestHeaders(ctx, config), + ...(await proxyBody(ctx)), + redirect: 'manual', + signal: controller.signal + }); + } finally { + clearTimeout(timer); + } +} + +async function forward(ctx, config, target) { + const upstream = await originFetch(ctx, config, target); + const result = withoutHeaders(upstream.headers, [...PROXY_RESPONSE_DROP, ...nameList(config.proxy_strip_headers)]); + applyCacheLifetime(result, config.cache_lifetime); + if (config.only_proxy_404) result.set('quant-404-proxied', 'true'); + return new Response(upstream.body, { status: upstream.status, statusText: upstream.statusText, headers: result }); +} + +const listsStatus = (list, status) => Array.isArray(list) && list.map(String).includes(String(status)); + +// With static error pages on, a listed origin status gets the published error page. +function withErrorPage(state, ctx, config, response) { + if (!config.static_error_page || !listsStatus(config.static_error_page_status_codes, response.status)) return response; + const page = state.stages.errorPage(ctx, response.status); + if (!page) return response; + if (response.body) response.body.cancel().catch(() => {}); + return page; +} + +function proxyError(state, ctx, rule, why) { + state.log(`Rule ${ruleLabel(rule)} (proxy): ${why}.`); + const page = configOf(rule).static_error_page ? state.stages.errorPage(ctx, 502) : null; + return page || new Response(PROXY_ERROR, { status: 502, headers: { 'content-type': HTML } }); +} + +async function proxy(state, ctx, rule) { + const config = configOf(rule); + const found = config.only_proxy_404 ? state.stages.static(ctx) : null; + if (found) return found; + const target = proxyTarget(rule, ctx); + if (!target) return proxyError(state, ctx, rule, 'the destination is not a URL on the configured origin'); + try { + return withErrorPage(state, ctx, config, await forward(ctx, config, target)); + } catch (_err) { + return proxyError(state, ctx, rule, 'the origin did not answer'); + } +} + +// --- Dispatch --------------------------------------------------------------------- + +const ACTIONS = { + edge_fn: (state, ctx, rule) => state.stages.edgeFunction(ctx, rule), + edge_auth: (state, ctx, rule) => state.stages.edgeAuth(ctx, rule), + redirect, + headers, + custom_response: customResponse, + auth: basicAuth, + proxy +}; + +/** + * Run one matched rule of the rule stage. + * `state.stages` gives the server stages: edgeFunction(ctx, rule), + * edgeAuth(ctx, rule), static(ctx) and errorPage(ctx, status). + * @returns {Promise }>} + * A Response ends the request; null or { continue } goes on to the next rule. + * An action that quant serve does not run gives null (start-up warns). + */ +export async function runAction(state, ctx, rule) { + const action = Object.hasOwn(ACTIONS, rule.action) ? ACTIONS[rule.action] : null; + return action ? action(state, ctx, rule) : null; +} + +function proxyWarnings(rule) { + const config = configOf(rule); + return [ + ...(config.waf_enabled ? [`Rule ${ruleLabel(rule)}: WAF settings are ignored in quant serve.`] : []), + ...(config.host ? [`Rule ${ruleLabel(rule)}: the host setting is ignored in quant serve. The origin gets the host of the to URL.`] : []) + ]; +} + +function ruleActionWarnings(rule) { + if (!SUPPORTED.has(rule.action)) return [`Rule ${ruleLabel(rule)} (${rule.action}) is not available in quant serve and is skipped.`]; + return rule.action === 'proxy' ? proxyWarnings(rule) : []; +} + +/** The start-up warnings for rules that quant serve skips in part or whole: one for each rule id. */ +export function actionWarnings(rules) { + const seen = new Set(); + return rules.flatMap((rule) => { + if (rule.disabled || seen.has(rule.id)) return []; + seen.add(rule.id); + return ruleActionWarnings(rule); + }); +} diff --git a/src/serve/rules.js b/src/serve/rules.js index 7783375..4ec18b4 100644 --- a/src/serve/rules.js +++ b/src/serve/rules.js @@ -34,10 +34,14 @@ function compile(pattern) { return re; } +// The text that a pattern is tested against: path plus query for a pattern with ?. +function subjects(pattern, pathname, search) { + return pattern.includes('?') ? [pathname + search] : pathForms(pathname); +} + function urlMatches(pattern, pathname, search) { const re = compile(pattern); - if (pattern.includes('?')) return re.test(pathname + search); - return pathForms(pathname).some(form => re.test(form)); + return subjects(pattern, pathname, search).some(form => re.test(form)); } function hostMatches(domains, host) { @@ -52,24 +56,46 @@ function ruleMatches(rule, ctx) { return rule.url.some(pattern => urlMatches(pattern, ctx.pathname, ctx.search || '')); } -function firstMatch(rules, ctx, wantFilter) { - return rules.find(rule => (rule.type === 'filter') === wantFilter && ruleMatches(rule, ctx)) || null; -} +// Filter and error page rules are not run in the rule stage. +const NOT_IN_RULE_STAGE = new Set(['edge_filter', 'error_page']); +const inRuleStage = rule => !NOT_IN_RULE_STAGE.has(rule.action); -/** The first enabled function or auth rule that matches, or null. Filter rules are skipped. */ +/** The first enabled rule of the rule stage that matches, or null. Filter and error page rules are skipped. */ export function matchRule(rules, ctx) { - return firstMatch(rules, ctx, false); + return rules.find(rule => inRuleStage(rule) && ruleMatches(rule, ctx)) || null; } /** - * Every enabled function or auth rule that matches, in run order. An auth - * rule's 201 continues to the next rule in this list. + * Every enabled rule of the rule stage that matches, in run order. A rule + * that continues (an auth 201, a headers rule) goes on to the next rule. */ export function matchRules(rules, ctx) { - return rules.filter(rule => rule.type !== 'filter' && ruleMatches(rule, ctx)); + return rules.filter(rule => inRuleStage(rule) && ruleMatches(rule, ctx)); } /** The first enabled filter rule that matches, or null. */ export function matchFilter(rules, ctx) { - return firstMatch(rules, ctx, true); + return rules.find(rule => rule.action === 'edge_filter' && ruleMatches(rule, ctx)) || null; +} + +const listsStatus = (rule, status) => (rule.config?.status_codes || []).map(String).includes(String(status)); + +/** The first enabled error page rule for the status that matches, or null. */ +export function matchErrorPage(rules, ctx, status) { + return rules.find(rule => rule.action === 'error_page' && listsStatus(rule, status) && ruleMatches(rule, ctx)) || null; +} + +/** + * The text that each * of the first matching URL pattern matched, and + * whether that pattern contains ?. Null when no pattern matches. + */ +export function ruleCaptures(rule, ctx) { + for (const pattern of rule.url) { + const re = compile(pattern); + for (const form of subjects(pattern, ctx.pathname, ctx.search || '')) { + const found = re.exec(form); + if (found) return { groups: found.slice(1), withQuery: pattern.includes('?') }; + } + } + return null; } diff --git a/src/serve/server.js b/src/serve/server.js index b68ccb3..c8f4684 100644 --- a/src/serve/server.js +++ b/src/serve/server.js @@ -19,16 +19,14 @@ import { applyInline, isHtmlFile, refusal } from './inline.js'; import { loadProject, watchProject } from './project.js'; import { LEDGERS_FILE, declareLedgers, readLedgerFile } from './kv/ledgers.js'; import { basicAuthResponse, globalHeaderObject, trailingSlashRedirect, withGlobalHeaders } from './project-config.js'; -import { matchRules, matchFilter } from './rules.js'; +import { PROXY_RESPONSE_DROP, proxyBody, proxyRequestHeaders, withoutHeaders } from './forward.js'; +import { runAction } from './rule-actions.js'; +import { matchErrorPage, matchRules, matchFilter } from './rules.js'; import { startWatchdog } from './watchdog.js'; import { buildIndex, containsProject, findOnDisk, lookup, serveStatic, notFound, NOT_SERVED } from './static.js'; const SERVER_HEADER = 'QuantCDN (quant serve)'; const AUTH_UNAVAILABLE = 'Authentication for this page is not available.'; -const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'expect']; -const PROXY_REQUEST_DROP = [...HOP_BY_HOP, 'host', 'content-length']; -// Node fetch decodes the body, so the encoding and length headers are dropped. -const PROXY_RESPONSE_DROP = [...HOP_BY_HOP, 'content-encoding', 'content-length']; const VERSION = readVersion(); function readVersion() { @@ -179,7 +177,7 @@ function requestContext(state, req) { remoteAddress: req.socket.remoteAddress, bodyPending: null, bodyStreamed: false, - authHeaders: null, + ruleHeaders: null, inlinePage: false }; } @@ -217,8 +215,9 @@ async function functionContext(state, ctx, mode) { // --- Rules ------------------------------------------------------------------- -// The headers of each auth 201 in a chain: a later auth function replaces -// a header that an earlier one set, and set-cookie values add up. +// The headers that rules add to the final response (an auth 201, a headers +// rule): a later rule replaces a header that an earlier one set, and +// set-cookie values add up. function mergeHeaders(target, source) { const headers = new Headers(target || []); for (const [key, value] of source) { @@ -232,7 +231,7 @@ async function authStage(state, ctx, fn) { if (!fn) return text(503, AUTH_UNAVAILABLE, { 'cache-control': 'no-store' }); const response = await state.executor.run(fn, await functionContext(state, ctx, 'auth')); if (response.status !== 201) return response; - ctx.authHeaders = mergeHeaders(ctx.authHeaders, authContinueHeaders(response)); + ctx.ruleHeaders = mergeHeaders(ctx.ruleHeaders, authContinueHeaders(response)); return null; } @@ -244,31 +243,20 @@ async function functionStage(state, ctx, rule, fn) { return state.executor.run(fn, { ...(await functionContext(state, ctx, 'function')), baseHeaders: globalHeaderObject(state.project.settings) }); } -// The matching rules run in order. An auth 201 continues to the next rule; -// any other auth response, and every function rule, ends the request. +// The matching rules run in order (see rule-actions.js). An auth 201, right +// basic-auth credentials and a headers rule continue to the next rule; any +// other response ends the request. async function ruleStage(state, ctx) { for (const rule of matchRules(state.project.rules, matchContext(ctx))) { - const fn = state.project.functions.get(rule.fnUuid); - const response = rule.type === 'auth' ? await authStage(state, ctx, fn) : await functionStage(state, ctx, rule, fn); - if (response) return response; + const result = await runAction(state, ctx, rule); + if (result instanceof Response) return result; + if (result && result.continue) ctx.ruleHeaders = mergeHeaders(ctx.ruleHeaders, result.headers); } return null; } // --- Proxy ------------------------------------------------------------------- -function withoutHeaders(source, names) { - const headers = new Headers(source); - for (const name of names) headers.delete(name); - return headers; -} - -function proxyRequestHeaders(req) { - const headers = new Headers(); - for (let i = 0; i < req.rawHeaders.length; i += 2) headers.append(req.rawHeaders[i], req.rawHeaders[i + 1]); - return withoutHeaders(headers, PROXY_REQUEST_DROP); -} - function proxyTarget(base, ctx) { const target = new URL(base); target.pathname = ctx.sentPath; @@ -276,19 +264,6 @@ function proxyTarget(base, ctx) { return target; } -// The proxy streams the request body to the dev server, unless a function -// (an auth rule that continued) has read it already. Then the proxy sends the -// capped body that the function saw. -async function proxyBody(ctx) { - if (['GET', 'HEAD'].includes(ctx.method)) return {}; - if (ctx.bodyPending) { - const bytes = await ctx.bodyPending; - return bytes ? { body: bytes } : {}; - } - ctx.bodyStreamed = true; - return { body: ctx.req, duplex: 'half' }; -} - async function proxyStage(state, ctx) { const base = state.options.proxy; if (!base) return null; @@ -338,7 +313,23 @@ async function devServerStage(state, ctx) { return response; } -function notFoundStage(state) { +// The published page for an error status: the page of the first matching +// error page rule that lists the status, else /_quant. Null when +// neither is published. +function errorPageStage(state, ctx, status) { + const rule = matchErrorPage(state.project.rules, matchContext(ctx), status); + const file = (rule && pageFile(state, rule.config.error_page_path)) || pageFile(state, `/_quant${status}`); + return file ? serveStatic(file, 'GET', null, status) : null; +} + +function pageFile(state, pagePath) { + const pathname = typeof pagePath === 'string' && pagePath.startsWith('/') ? canonicalPath(pagePath) : null; + return pathname ? staticFile(state, { pathname, search: '' }) : null; +} + +function notFoundStage(state, ctx) { + const page = errorPageStage(state, ctx, 404); + if (page) return page; const { response, warning } = notFound(state.dir, state.index); if (warning && !state.notFoundWarned) { state.notFoundWarned = true; @@ -391,10 +382,10 @@ async function siteResponse(state, ctx) { const denied = basicAuthResponse(settings, ctx.req.headers.authorization); if (denied) return denied; const ruled = await ruleStage(state, ctx); - if (ruled) return filterStage(state, ctx, withGlobalHeaders(ruled, settings, false)); + if (ruled) return inlineStage(state, ctx, await filterStage(state, ctx, withGlobalHeaders(ruled, settings, false))); const redirect = trailingSlashRedirect(settings, ctx.sentPath, ctx.search); if (redirect) return redirect; - const response = await devServerStage(state, ctx) || staticStage(state, ctx) || notFoundStage(state); + const response = await devServerStage(state, ctx) || staticStage(state, ctx) || notFoundStage(state, ctx); return inlineStage(state, ctx, await filterStage(state, ctx, withGlobalHeaders(response, settings))); } @@ -413,9 +404,9 @@ async function respond(state, ctx) { // --- Headers and send ------------------------------------------------------------ -function finalHeaders(state, source, authHeaders) { +function finalHeaders(state, source, ruleHeaders) { const headers = new Headers(source); - for (const [key, value] of authHeaders || []) { + for (const [key, value] of ruleHeaders || []) { if (key === 'set-cookie') headers.append(key, value); else headers.set(key, value); } @@ -435,8 +426,8 @@ function writeHead(res, response, headers) { if (cookies.length > 0) res.setHeader('set-cookie', cookies); } -async function send(state, res, response, method, authHeaders) { - writeHead(res, response, finalHeaders(state, response.headers, authHeaders)); +async function send(state, res, response, method, ruleHeaders) { + writeHead(res, response, finalHeaders(state, response.headers, ruleHeaders)); if (!response.body || method === 'HEAD') { res.end(); if (response.body) await response.body.cancel().catch(() => {}); @@ -474,7 +465,7 @@ async function handleRequest(state, req, res) { response = new Response(null, { status: 500 }); } try { - await send(state, res, response, req.method, ctx && ctx.authHeaders); + await send(state, res, response, req.method, ctx && ctx.ruleHeaders); } catch (err) { sendFailed(state, req, res, err); } @@ -510,16 +501,18 @@ function upgradeAllowed(state, req) { } } -// The auth rules of the upgrade path run in order, as for a request. The -// response of an auth that does not answer 201 ends the upgrade. Function -// rules do not run for an upgrade. An upgrade has no body. +// The auth and basic-auth rules of the upgrade path run in order, as for a +// request. The response of an auth that does not continue ends the upgrade. +// Other rules do not run for an upgrade. An upgrade has no body. +const UPGRADE_ACTIONS = new Set(['edge_auth', 'auth']); + async function upgradeAuth(state, req) { const denied = basicAuthResponse(state.project.settings, req.headers.authorization); if (denied) return denied; const ctx = { ...requestContext(state, req), bodyStreamed: true }; for (const rule of matchRules(state.project.rules, matchContext(ctx))) { - if (rule.type !== 'auth') continue; - const response = await authStage(state, ctx, state.project.functions.get(rule.fnUuid)); + if (!UPGRADE_ACTIONS.has(rule.action)) continue; + const response = await runAction(state, ctx, rule); if (response) return response; } return null; @@ -572,8 +565,8 @@ async function handleUpgrade(state, req, socket, head) { // --- Start-up ------------------------------------------------------------------- function projectFiles(state) { - const { functions, rules, meta, fixtures } = state.options; - return { root: state.root, functions, rules, meta, fixtures }; + const { functions, rules, meta, fixtures, runtimeRules } = state.options; + return { root: state.root, functions, rules, meta, fixtures, runtimeRules }; } function siteIndex(state) { @@ -640,6 +633,17 @@ function storeList(state) { return state.project.stores.map((store) => ({ name: store.name, id: storeId(state.org, state.projectName, store.name) })); } +// The server stages that the rule actions call (see rule-actions.js). +function serverStages(state) { + const fnOf = (rule) => state.project.functions.get(rule.fnUuid); + return { + edgeFunction: (ctx, rule) => functionStage(state, ctx, rule, fnOf(rule)), + edgeAuth: (ctx, rule) => authStage(state, ctx, fnOf(rule)), + static: (ctx) => staticStage(state, ctx), + errorPage: (ctx, status) => errorPageStage(state, ctx, status) + }; +} + function createState(options) { const root = path.resolve(options.root || process.cwd()); const state = { @@ -654,6 +658,7 @@ function createState(options) { allowedHosts: allowedHosts(options) }; checkIdentity(state.org, state.projectName); + state.stages = serverStages(state); state.project = loadProject(projectFiles(state)); state.executor = createExecutor({ kvStore: options.kvStore, org: state.org, project: state.projectName, root, @@ -704,7 +709,8 @@ function closer(state, server, cleanups) { * @param {Object} options The parsed flags plus root and kvStore: * root, dir, port, host, functions, rules, meta, fixtures, org, project, * domain, proxy, watch, geo, clientIp, timeoutMs, kvStore, log, onSourceChange, - * backend (the local KV backend, for the control route) and clock ('real' or 'manual'). + * backend (the local KV backend, for the control route), clock ('real' or 'manual') + * and runtimeRules (runtime rules that run with the rules of the rules manifest, by weight). * @returns {Promise<{ url: string, close: () => Promise, setRunPid: (pid: number) => void, summary: Object }>} * setRunPid: the --run process group that the watchdog ends when a function blocks the server. * summary: { stores, rules, libraries, warnings } for the start-up output. @@ -727,7 +733,7 @@ export async function startServer(options) { setRunPid: (pid) => state.watchdog.setRunPid(pid), summary: { stores: storeList(state), - rules: state.project.rules.map((rule) => ({ id: rule.id, type: rule.type, urls: rule.url })), + rules: state.project.rules.map((rule) => ({ id: rule.id, type: rule.type || rule.action, urls: rule.url })), libraries: fromProject, warnings: [site.warning, ...libraryErrors, ...state.project.warnings].filter(Boolean) } diff --git a/tests/unit/helper/edge-rules-manifest.test.mjs b/tests/unit/helper/edge-rules-manifest.test.mjs index c5bfc45..4fd2265 100644 --- a/tests/unit/helper/edge-rules-manifest.test.mjs +++ b/tests/unit/helper/edge-rules-manifest.test.mjs @@ -73,6 +73,13 @@ describe('Edge rules manifest helpers', () => { expect(runtime.fnUuid).to.equal(functions.find(fn => fn.id === 'orbit-api-v1').uuid.toLowerCase()); }); + it('sets the runtime action from the rule type', () => { + expect(toRuntimeRule(manifest().rules[0])).to.include({ action: 'edge_fn' }); + expect(toRuntimeRule(manifest({ type: 'auth' }).rules[0])).to.include({ action: 'edge_auth' }); + expect(toRuntimeRule(manifest({ type: 'filter' }).rules[0])).to.include({ action: 'edge_filter' }); + expect(toRuntimeRule(manifest().rules[0]).config).to.deep.equal({}); + }); + it('lowercases the function uuid and keeps explicit matchers', () => { const rule = toRuntimeRule({ ...manifest({ methods: ['GET'], domains: ['a.com'], disabled: true, function_uuid: UUID.toUpperCase() }).rules[0] }); expect(rule).to.include({ fnUuid: UUID, disabled: true }); diff --git a/tests/unit/serve/rule-actions.test.mjs b/tests/unit/serve/rule-actions.test.mjs new file mode 100644 index 0000000..03199eb --- /dev/null +++ b/tests/unit/serve/rule-actions.test.mjs @@ -0,0 +1,380 @@ +import { expect } from 'chai'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { startServer } from '../../../src/serve/server.js'; +import { actionWarnings } from '../../../src/serve/rule-actions.js'; + +const PASSWORD = 'pa55-never-printed'; +const SECRET = 'origin-secret-never-printed'; + +let servers; +let upstreams; +let lines; +let tmp; + +// A runtime rule as the rule stage reads it. +function rule(action, url, config = {}, over = {}) { + return { id: `${action}-rule`, name: `${action} rule`, action, config, url, domain: ['any'], methods: null, weight: 0, disabled: false, ...over }; +} + +function project(files = {}) { + const root = fs.mkdtempSync(path.join(tmp, 'project-')); + fs.mkdirSync(path.join(root, 'dist'), { recursive: true }); + for (const [name, body] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, 'dist', name)), { recursive: true }); + fs.writeFileSync(path.join(root, 'dist', name), body); + } + return root; +} + +async function start(runtimeRules, files = {}) { + const server = await startServer({ + root: project(files), dir: 'dist', port: 0, host: '127.0.0.1', org: 'acme', project: 'site', + runtimeRules, log: (line) => lines.push(line) + }); + servers.push(server); + return server; +} + +function get(server, pathAndQuery, headers = {}) { + return fetch(`${server.url}${pathAndQuery}`, { headers, redirect: 'manual' }); +} + +function basic(user, pass) { + return `Basic ${Buffer.from(`${user}:${pass}`).toString('base64')}`; +} + +// A local origin that records each request and answers with handler(req). +async function origin(handler) { + const seen = []; + const server = http.createServer((req, res) => { + seen.push({ url: req.url, headers: req.headers, method: req.method }); + handler(req, res); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + upstreams.push(server); + return { url: `http://127.0.0.1:${server.address().port}`, seen }; +} + +function echo(req, res) { + if (req.url.startsWith('/moved')) { + res.writeHead(302, { location: '/elsewhere' }); + res.end(); + return; + } + res.writeHead(201, { 'content-type': 'text/plain', 'x-origin': 'yes', 'x-internal': 'drop-me', connection: 'close' }); + res.end(`origin saw ${req.url}`); +} + +describe('serve rule actions', () => { + beforeEach(() => { + servers = []; + upstreams = []; + lines = []; + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-serve-actions-')); + }); + + afterEach(async () => { + await Promise.all(servers.map((server) => server.close())); + await Promise.all(upstreams.map((server) => new Promise((resolve) => { + server.closeAllConnections(); + server.close(resolve); + }))); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + describe('redirect', () => { + it('substitutes $1 from the URL pattern and adds the query string', async () => { + const server = await start([rule('redirect', ['/blog/*'], { to: '/news/$1', status_code: 302 })]); + const res = await get(server, '/blog/2026/first-post?page=2'); + expect(res.status).to.equal(302); + expect(res.headers.get('location')).to.equal('/news/2026/first-post?page=2'); + }); + + it('substitutes each * in order', async () => { + const server = await start([rule('redirect', ['/blogs/*/category/*'], { to: '/categories/$2/$1' })]); + const res = await get(server, '/blogs/ada/category/maths'); + expect(res.headers.get('location')).to.equal('/categories/maths/ada'); + }); + + it('defaults to 301 and substitutes $* with the whole path', async () => { + const server = await start([rule('redirect', ['/old'], { to: 'https://www.example.com$*' })], { 'old/index.html': 'old' }); + const res = await get(server, '/old'); + expect(res.status).to.equal(301); + expect(res.headers.get('location')).to.equal('https://www.example.com/old'); + }); + + it('keeps $1 as literal text when the pattern has no *', async () => { + const server = await start([rule('redirect', ['/promo'], { to: '/news/$1', status_code: '308' })]); + const res = await get(server, '/promo'); + expect(res.status).to.equal(308); + expect(res.headers.get('location')).to.equal('/news/$1'); + }); + + it('does not add the query string when the pattern contains ?', async () => { + const server = await start([rule('redirect', ['/search?q=*'], { to: '/find/$1' })]); + const res = await get(server, '/search?q=cats'); + expect(res.headers.get('location')).to.equal('/find/cats'); + }); + }); + + describe('headers', () => { + it('adds the headers of two rules to the static file, the later rule replacing a header', async () => { + const server = await start([ + rule('headers', ['*'], { headers: { 'x-first': 'one', 'x-shared': 'first', 'set-cookie': 'a=1' } }, { id: 'h1' }), + rule('headers', ['/about*'], { headers: { 'x-second': 'two', 'x-shared': 'second', 'set-cookie': 'b=2' } }, { id: 'h2', weight: 1 }) + ], { 'about/index.html': '

About

' }); + const res = await get(server, '/about/'); + expect(res.status).to.equal(200); + expect(await res.text()).to.equal('

About

'); + expect(res.headers.get('x-first')).to.equal('one'); + expect(res.headers.get('x-second')).to.equal('two'); + expect(res.headers.get('x-shared')).to.equal('second'); + expect(res.headers.getSetCookie()).to.deep.equal(['a=1', 'b=2']); + }); + + it('cannot set the quant-server header', async () => { + const server = await start([rule('headers', ['*'], { headers: { 'quant-server': 'other', age: '9' } })], { 'index.html': 'home' }); + const res = await get(server, '/'); + expect(res.headers.get('quant-server')).to.equal('QuantCDN (quant serve)'); + expect(res.headers.get('age')).to.equal(null); + }); + + it('adds the headers to a response of a rule below it', async () => { + const server = await start([ + rule('headers', ['*'], { headers: { 'x-policy': 'strict' } }), + rule('redirect', ['/old'], { to: '/new' }, { weight: 1 }) + ]); + const res = await get(server, '/old'); + expect(res.status).to.equal(301); + expect(res.headers.get('x-policy')).to.equal('strict'); + }); + }); + + describe('custom response', () => { + it('answers with the status and body, as text/html', async () => { + const server = await start([rule('custom_response', ['/gone'], { custom_response_status_code: 410, custom_response_body: '

Gone

' })]); + const res = await get(server, '/gone'); + expect(res.status).to.equal(410); + expect(res.headers.get('content-type')).to.equal('text/html; charset=utf-8'); + expect(await res.text()).to.equal('

Gone

'); + }); + + it('takes the content type from a headers rule above it', async () => { + const server = await start([ + rule('headers', ['/health'], { headers: { 'content-type': 'application/json' } }), + rule('custom_response', ['/health'], { custom_response_status_code: '200', custom_response_body: '{"ok":true}' }, { weight: 1 }) + ]); + const res = await get(server, '/health'); + expect(res.status).to.equal(200); + expect(res.headers.get('content-type')).to.equal('application/json'); + expect(await res.json()).to.deep.equal({ ok: true }); + }); + }); + + describe('basic auth', () => { + const files = { 'staff/index.html': '

Staff

' }; + const auth = () => rule('auth', ['/staff*'], { auth_user: 'ada', auth_pass: PASSWORD }); + + it('answers 401 with a Basic challenge when the credentials are missing', async () => { + const server = await start([auth()], files); + const res = await get(server, '/staff/'); + expect(res.status).to.equal(401); + expect(res.headers.get('www-authenticate')).to.match(/^Basic realm=/); + expect(await res.text()).to.not.include('Staff'); + }); + + it('answers 401 when the credentials are wrong', async () => { + const server = await start([auth()], files); + const res = await get(server, '/staff/', { authorization: basic('ada', 'wrong') }); + expect(res.status).to.equal(401); + }); + + it('continues to the static file when the credentials are right', async () => { + const server = await start([auth()], files); + const res = await get(server, '/staff/', { authorization: basic('ada', PASSWORD) }); + expect(res.status).to.equal(200); + expect(await res.text()).to.equal('

Staff

'); + }); + }); + + describe('error page', () => { + it('serves the configured page with the 404 status for a missing static path', async () => { + const server = await start([ + rule('error_page', ['/fr/*'], { status_codes: ['404'], error_page_path: '/fr/_quant404' }) + ], { 'fr/_quant404/index.html': '

Introuvable

', '_quant404/index.html': '

Not found

' }); + const res = await get(server, '/fr/missing'); + expect(res.status).to.equal(404); + expect(await res.text()).to.equal('

Introuvable

'); + }); + + it('is not run in the rule stage', async () => { + const server = await start([rule('error_page', ['*'], { status_codes: ['404'], error_page_path: '/oops' })], { 'index.html': 'home', 'oops/index.html': 'oops' }); + const res = await get(server, '/'); + expect(res.status).to.equal(200); + expect(await res.text()).to.equal('home'); + }); + + it('uses /_quant404 for a rule of another status or a page that is not published', async () => { + const server = await start([ + rule('error_page', ['*'], { status_codes: ['500'], error_page_path: '/oops' }, { id: 'e1' }), + rule('error_page', ['/b/*'], { status_codes: ['404'], error_page_path: '/not-published' }, { id: 'e2', weight: 1 }) + ], { 'oops/index.html': 'oops', '_quant404/index.html': '

Not found

' }); + expect(await (await get(server, '/a/x')).text()).to.equal('

Not found

'); + const res = await get(server, '/b/x'); + expect(res.status).to.equal(404); + expect(await res.text()).to.equal('

Not found

'); + }); + }); + + describe('proxy', () => { + it('forwards the path and query to the origin and returns its status, body and headers', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['/api/*'], { to: `${up.url}$*` })]); + const res = await get(server, '/api/v1/users?page=2'); + expect(res.status).to.equal(201); + expect(await res.text()).to.equal('origin saw /api/v1/users?page=2'); + expect(res.headers.get('x-origin')).to.equal('yes'); + expect(res.headers.get('connection')).to.not.equal('close'); + expect(up.seen[0].headers.host).to.equal(new URL(up.url).host); + }); + + it('sends every request to the origin / when to has no placeholder', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['/api/*'], { to: up.url })]); + expect(await (await get(server, '/api/x?y=1')).text()).to.equal('origin saw /?y=1'); + }); + + it('does not follow an origin redirect', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['*'], { to: `${up.url}$*` })]); + const res = await get(server, '/moved'); + expect(res.status).to.equal(302); + expect(res.headers.get('location')).to.equal('/elsewhere'); + expect(up.seen).to.have.length(1); + }); + + it('sends injected headers and origin credentials, and strips headers', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['*'], { + to: `${up.url}$*`, + auth_user: 'svc', + auth_pass: PASSWORD, + inject_headers: { 'x-shared-secret': SECRET }, + proxy_strip_request_headers: ['x-drop'], + proxy_strip_headers: ['x-internal'] + })]); + const res = await get(server, '/p', { 'x-drop': '1', 'x-keep': '1' }); + expect(res.headers.get('x-internal')).to.equal(null); + const seen = up.seen[0].headers; + expect(seen['x-shared-secret']).to.equal(SECRET); + expect(seen.authorization).to.equal(basic('svc', PASSWORD)); + expect(seen['x-drop']).to.equal(undefined); + expect(seen['x-keep']).to.equal('1'); + }); + + it('sets cache-control from cache_lifetime', async () => { + const up = await origin(echo); + const server = await start([ + rule('proxy', ['/none'], { to: `${up.url}$*`, cache_lifetime: '0' }, { id: 'p0' }), + rule('proxy', ['/ten'], { to: `${up.url}$*`, cache_lifetime: '600' }, { id: 'p1' }) + ]); + expect((await get(server, '/none')).headers.get('cache-control')).to.equal('no-store'); + expect((await get(server, '/ten')).headers.get('cache-control')).to.equal('public, max-age=600'); + }); + + it('answers the 502 Proxy Error page when the origin does not answer', async () => { + const up = await origin(echo); + const dead = up.url; + await new Promise((resolve) => upstreams.pop().close(resolve)); + const server = await start([rule('proxy', ['*'], { to: `${dead}$*` })]); + const res = await get(server, '/x'); + expect(res.status).to.equal(502); + expect(await res.text()).to.include('Proxy Error'); + }); + + it('serves /_quant502 when the rule uses static error pages', async () => { + const up = await origin((req, res) => { res.writeHead(503); res.end('origin down'); }); + const server = await start([rule('proxy', ['/app/*'], { + to: `${up.url}$*`, static_error_page: 'on', static_error_page_status_codes: ['503'] + })], { '_quant503/index.html': '

Back soon

' }); + const res = await get(server, '/app/x'); + expect(res.status).to.equal(503); + expect(await res.text()).to.equal('

Back soon

'); + }); + + it('never sends the request to a host other than the configured origin', async () => { + const up = await origin(echo); + const other = await origin(echo); + const otherHost = new URL(other.url).host; + const server = await start([rule('proxy', ['/p/*'], { to: `${up.url}$1` })]); + const res = await get(server, `/p/@${otherHost}/x`); + expect(res.status).to.equal(502); + expect(up.seen).to.have.length(0); + expect(other.seen).to.have.length(0); + }); + + it('serves the static file for an only_proxy_404 rule when the path exists, else proxies', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['*'], { to: `${up.url}$*`, only_proxy_404: true })], { 'here/index.html': 'static here' }); + expect(await (await get(server, '/here/')).text()).to.equal('static here'); + const res = await get(server, '/gone'); + expect(await res.text()).to.equal('origin saw /gone'); + expect(res.headers.get('quant-404-proxied')).to.equal('true'); + }); + }); + + describe('rules that quant serve does not run', () => { + it('skips the rule with one warning per rule, and warns once about WAF settings', async () => { + const up = await origin(echo); + const rules = [ + rule('bot_challenge', ['*'], {}, { id: 'bc', name: 'Challenge' }), + rule('serve_static', ['*'], {}, { id: 'ss', name: null }), + rule('proxy', ['/api/*'], { to: `${up.url}$*`, waf_enabled: true, waf_config: { mode: 'block' } }, { id: 'px', name: 'API' }) + ]; + const server = await start(rules, { 'index.html': 'home' }); + const warnings = server.summary.warnings; + expect(warnings.filter((w) => w === 'Rule Challenge (bot_challenge) is not available in quant serve and is skipped.')).to.have.length(1); + expect(warnings.filter((w) => w === 'Rule ss (serve_static) is not available in quant serve and is skipped.')).to.have.length(1); + expect(warnings.filter((w) => w === 'Rule API: WAF settings are ignored in quant serve.')).to.have.length(1); + expect(await (await get(server, '/')).text()).to.equal('home'); + expect((await get(server, '/api/x')).status).to.equal(201); + }); + + it('warns that the host setting of a proxy rule is not used', async () => { + const up = await origin(echo); + const server = await start([rule('proxy', ['*'], { to: `${up.url}$*`, host: 'origin.example.com' }, { id: 'px', name: 'API' })]); + expect(server.summary.warnings).to.include('Rule API: the host setting is ignored in quant serve. The origin gets the host of the to URL.'); + expect(server.summary.warnings.join('\n')).to.not.include('origin.example.com'); + await get(server, '/x'); + expect(up.seen[0].headers.host).to.equal(new URL(up.url).host); + }); + + it('gives one warning for each rule id', () => { + const twice = rule('bot_challenge', ['*'], {}, { id: 'bc' }); + expect(actionWarnings([twice, twice])).to.deep.equal(['Rule bot_challenge rule (bot_challenge) is not available in quant serve and is skipped.']); + }); + }); + + describe('credentials', () => { + it('never prints a basic-auth password or a proxy header value', async () => { + const rules = [ + rule('auth', ['/staff*'], { auth_user: 'ada', auth_pass: PASSWORD }), + rule('proxy', ['/api/*'], { to: 'http://127.0.0.1:1$*', auth_pass: PASSWORD, inject_headers: { 'x-shared-secret': SECRET, 'x-bad': `bad\n${SECRET}` }, waf_enabled: true }, { id: 'px' }), + rule('headers', ['/h'], { headers: { 'x-bad': `bad\n${SECRET}` } }, { id: 'hb' }), + rule('redirect', ['/r'], { to: `/x\n${SECRET}` }, { id: 'rb' }), + rule('mystery', ['*'], { secret: SECRET }, { id: 'my' }) + ]; + const server = await start(rules); + await get(server, '/staff', { authorization: basic('ada', 'wrong') }); + await get(server, '/api/x'); + await get(server, '/h'); + await get(server, '/r'); + const printed = [...lines, ...server.summary.warnings].join('\n'); + expect(printed).to.not.include(PASSWORD); + expect(printed).to.not.include(SECRET); + }); + }); +}); diff --git a/tests/unit/serve/rules.test.mjs b/tests/unit/serve/rules.test.mjs index 980c1db..0d425c7 100644 --- a/tests/unit/serve/rules.test.mjs +++ b/tests/unit/serve/rules.test.mjs @@ -1,9 +1,11 @@ import { expect } from 'chai'; -import { orderRules, matchRule, matchFilter, pathForms } from '../../../src/serve/rules.js'; +import { orderRules, matchRule, matchRules, matchFilter, matchErrorPage, pathForms, ruleCaptures } from '../../../src/serve/rules.js'; -const rule = (over = {}) => ({ - id: 'r', type: 'function', url: ['/api/*'], domain: ['any'], methods: null, weight: 0, disabled: false, fnUuid: 'u', ...over -}); +const ACTION = { function: 'edge_fn', auth: 'edge_auth', filter: 'edge_filter' }; +const rule = (over = {}) => { + const base = { id: 'r', type: 'function', url: ['/api/*'], domain: ['any'], methods: null, weight: 0, disabled: false, fnUuid: 'u', ...over }; + return { action: ACTION[base.type], ...base }; +}; const ctx = (over = {}) => ({ host: 'example.com', method: 'GET', pathname: '/api/projects', search: '', ...over }); describe('serve rules', () => { @@ -89,4 +91,33 @@ describe('serve rules', () => { expect(matchFilter([fn], ctx())).to.equal(null); }); }); + + describe('error page rules', () => { + const page = (over = {}) => rule({ id: 'e', action: 'error_page', url: ['*'], config: { status_codes: ['404'], error_page_path: '/p' }, ...over }); + + it('are not in the rule stage', () => { + expect(matchRule([page()], ctx())).to.equal(null); + expect(matchRules([page()], ctx())).to.deep.equal([]); + }); + + it('match only for a listed status', () => { + const e = page(); + expect(matchErrorPage([e], ctx(), 404)).to.equal(e); + expect(matchErrorPage([e], ctx(), 500)).to.equal(null); + expect(matchErrorPage([page({ disabled: true })], ctx(), 404)).to.equal(null); + }); + }); + + describe('ruleCaptures', () => { + it('returns the text of each * from the first matching pattern', () => { + const r = rule({ url: ['/none/*', '/blogs/*/category/*'] }); + expect(ruleCaptures(r, ctx({ pathname: '/Blogs/Ada/category/maths' }))).to.deep.equal({ groups: ['Ada', 'maths'], withQuery: false }); + expect(ruleCaptures(r, ctx({ pathname: '/other' }))).to.equal(null); + }); + + it('matches a pattern with ? against the path and query', () => { + const r = rule({ url: ['/search?q=*'] }); + expect(ruleCaptures(r, ctx({ pathname: '/search', search: '?q=cats' }))).to.deep.equal({ groups: ['cats'], withQuery: true }); + }); + }); }); From 8ea2042528ad08add824b165b28259326c479710 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:18:26 +1000 Subject: [PATCH 16/32] feat(serve): --pull-rules runs the portal rules in edge order --- README.md | 26 +- src/commands/serve.js | 53 +++- src/serve/portal/pull-rules.js | 174 ++++++++++++ src/serve/project-config.js | 31 +++ src/serve/project.js | 16 +- src/serve/rule-actions.js | 18 +- src/serve/rules.js | 13 +- src/serve/server.js | 7 +- tests/unit/commands/serve.test.mjs | 2 +- tests/unit/serve/pull-rules.test.mjs | 372 +++++++++++++++++++++++++ tests/unit/serve/rule-actions.test.mjs | 33 ++- tests/unit/serve/rules.test.mjs | 12 + 12 files changed, 732 insertions(+), 25 deletions(-) create mode 100644 src/serve/portal/pull-rules.js create mode 100644 tests/unit/serve/pull-rules.test.mjs diff --git a/README.md b/README.md index 1a3147b..90945a5 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,8 @@ Flags: - `--indexes` - Index state for the session: `active` or `building` (default: `active`). With `building`, the indexes of the stores stay in the building state. - `--clock` - `real` or `manual` (default: `real`). With `manual`, the time of the local KV stops at the start time and, in a session, moves only with `POST /_quant_serve/clock`. See [Local ledgers](#local-ledgers). - `--version-info` - Print the versions of the CLI and its KV and test packages, then exit. -- `--pull-rules`, `--kv-remote` - Not available yet. The command stops with a message. +- `--pull-rules` - Read the rules of the portal project one time at start, and run them with the rules of `quant/edge-rules.json`. See [Portal rules](#portal-rules---pull-rules). +- `--kv-remote` - Not available yet. The command stops with a message. A key that `initial_values` reveals once is shown in a box when the store is first seeded. Save the key then. Quant serve also writes it to a file under `.quant/kv/`. @@ -465,6 +466,29 @@ The order is the order of the edge: global headers, basic authentication, rules, `quant/serve.json` is local only. No deploy command reads it, and `quant deploy` never uploads it. `quant serve` never prints the password. Do not put a production password in the file. A file with the wrong shape stops `quant serve` at start-up, with a message that names the field. It also stops `quant kv --local` (exit code 1), because that command reads the `org`, `project` and `meta` defaults from the same file. With `--watch`, `quant serve` reads the file again when it changes. If the new file has the wrong shape, `quant serve` prints the message and keeps the last valid settings. +## Portal rules (--pull-rules) + +`quant serve --pull-rules` reads the rules of the portal project one time at start. The rules run locally in the order of the edge, with the rules of `quant/edge-rules.json`. + +```bash +export QUANT_API_TOKEN=... # a portal token with the rules:read scope +export QUANT_BASE_URL=https://dashboard.quantcdn.io/api/v2 +quant serve dist --pull-rules --org acme --project marketing-site +``` + +- The token and the portal URL come only from `QUANT_API_TOKEN` and `QUANT_BASE_URL`. The organisation and the project come from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. `--pull-rules` stops if no organisation or project is set. +- The token needs the `rules:read` scope. Without it, `quant serve` stops with a message that names the scope. +- The rules that a deploy of `quant/edge-rules.json` owns on the portal are replaced by the rules of the local file. Each local rule goes before the first portal rule with a higher weight. This is the order that the edge has after a rules deploy of the same file. +- The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of portal rules that the local file replaces. +- With `--watch`, a change to `quant/edge-rules.json` merges the local rules again with the portal rules in memory. `quant serve` does not read the portal again. +- A rule type that quant serve does not run (for example a bot challenge) is skipped, with one warning at start-up. A rule with a country, IP, ASN, user agent or traffic source condition is skipped, with one warning. +- A proxy rule with failover settings or with TLS verification off gets one warning: these settings work differently in quant serve. +- For some organisations the portal leaves some rule types out of the list. Then `quant serve` prints one warning, and those rules do not run locally. +- With the `projects:read` scope, the global headers of the portal project are used too. The `globalHeaders` of `quant/serve.json` override a portal header of the same name, with one warning for each name. Without the scope, `quant serve` prints one warning and uses only the local headers. +- Basic authentication and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. +- The pulled rules and headers stay in memory. `quant serve` does not write them to `.quant/` and does not print a setting value. +- A pulled function rule runs only when its function uuid is in `quant/edge-functions.json`. + ## Edge libraries in quant serve The Quant edge gives functions a set of library globals. In `quant serve`, each library global comes from your project. Install the package in the project, and the global uses it. diff --git a/src/commands/serve.js b/src/commands/serve.js index a3f50b5..8b4e786 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -15,9 +15,12 @@ import { DEFAULT_FILES, loadProject } from '../serve/project.js'; import { openLocalKv } from '../serve/kv/wire.js'; import { LEDGERS_FILE, readLedgerFile } from '../serve/kv/ledgers.js'; import { readServeConfig } from '../serve/project-config.js'; +import { createPortalClient, portalOptions } from '../serve/portal/client.js'; +import { pullProject } from '../serve/portal/pull-rules.js'; const DEFAULTS = Object.freeze({ dir: 'dist', port: 8787, host: '127.0.0.1', org: 'local', project: 'dev', userName: 'Owner' }); -const LATER_RELEASE = ['pull-rules', 'kv-remote']; +const LATER_RELEASE = ['kv-remote']; +const PULL_IDENTITY = '--pull-rules needs the portal organisation and project: pass --org and --project, or set them in quant/serve.json or quant.json.'; const CLOCKS = ['real', 'manual']; const LOOPBACK_HOSTS = ['127.0.0.1', '::1', 'localhost']; const POSIX = process.platform !== 'win32'; @@ -136,6 +139,17 @@ function identityOptions(args, root, gitUserName, local) { }; } +// --pull-rules reads the rules of a real project, so the org and project +// must be named (the local defaults are not a portal project). +function pullOptions(args, root, local) { + if (args['pull-rules'] !== true) return { pullRules: false }; + const saved = readQuantJson(root); + const org = first([args.org, args.clientid, local.org, saved.clientid], undefined); + const project = first([args.project, local.project, saved.project], undefined); + if (org === undefined || project === undefined) throw new Error(PULL_IDENTITY); + return { pullRules: true }; +} + function kvOptions(args) { return { reset: args.reset === true, @@ -172,6 +186,7 @@ export function resolveOptions(args, { root = process.cwd(), gitUserName = readG root, ...siteOptions(args, local), ...identityOptions(args, root, gitUserName, local), + ...pullOptions(args, root, local), ...kvOptions(args), ...devOptions(args) }; @@ -208,6 +223,10 @@ function ledgerLines(ledgers, clock) { return section(title, ledgers.map(({ store, defs }) => `${store} → ${defs.map((def) => def.name).join(', ')}`)); } +function pullLines(pull) { + return pull ? [`Pulled ${pull.count} portal rules (${pull.managed} from quant/edge-rules.json).`] : []; +} + function clockLines(clock) { return clock === 'manual' ? ['Clock: manual. Time moves only with POST /_quant_serve/clock.'] : []; } @@ -217,10 +236,11 @@ function clockLines(clock) { * rules, the edge libraries from the project, the warnings, then the * revealed keys in a box. */ -export function startupLines({ url, host, summary, keys = [], ledgers = [], clock = 'real' }) { +export function startupLines({ url, host, summary, keys = [], ledgers = [], clock = 'real', pull = null }) { const libraries = summary.libraries.length > 0 ? summary.libraries.join(', ') : 'none'; return [ `quant serve is running at ${url}`, + ...pullLines(pull), ...section('KV stores:', summary.stores.map((store) => `${store.name} → ${store.location || store.id}`)), ...ledgerLines(ledgers, clock), ...clockLines(clock), @@ -423,6 +443,20 @@ async function localKv(options, log, print) { }; } +// --pull-rules: read the portal rules and global headers one time. They +// stay in memory only. QUANT_API_TOKEN and QUANT_BASE_URL come from the +// environment; the org and project are the serve identity. +async function pulledProject(options, { env = process.env, fetchImpl } = {}) { + if (!options.pullRules) return null; + const { customer, project, token, base } = portalOptions({ clientid: options.org, project: options.project }, env); + const client = createPortalClient({ base, token, ...(fetchImpl ? { fetchImpl } : {}) }); + return pullProject(client, { customer, project }); +} + +function withPull(options, pulled) { + return pulled ? { ...options, pulledRules: pulled.rules, pulledHeaders: pulled.headers } : options; +} + function serverOptions(options, { kvStore, backend, log }, print) { return { ...options, @@ -484,14 +518,17 @@ function watchRun(server, child) { * command, print the start-up output, then serve until `stop` resolves * (default: SIGINT, SIGTERM or SIGHUP). * @param {Object} options From resolveOptions(). - * @param {Object} [deps] stop, print, log, kvStore (replaces the local KV), onListening. + * @param {Object} [deps] stop, print, log, kvStore (replaces the local KV), onListening, + * env (QUANT_API_TOKEN and QUANT_BASE_URL for --pull-rules) and fetchImpl (the portal fetch). */ -export async function runServe(options, { stop, print = console.log, log, kvStore, onListening } = {}) { +export async function runServe(options, { stop, print = console.log, log, kvStore, onListening, env, fetchImpl } = {}) { if (options.build) runBuild(options.build, options.root); + const pulled = await pulledProject(options, { env, fetchImpl }); const kv = await kvFor(options, kvStore, log, print); - const server = await startOrShowKeys(serverOptions(options, { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); - const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...server.summary.warnings] }; - const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, ...startupSummary(summary, kv) }); + const server = await startOrShowKeys(serverOptions(withPull(options, pulled), { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); + const pullWarnings = pulled ? pulled.warnings : []; + const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...pullWarnings, ...server.summary.warnings] }; + const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, pull: pulled, ...startupSummary(summary, kv) }); for (const line of lines) print(line); const child = options.run ? startRun(options.run, options.root, print, options.proxy) : null; watchRun(server, child); @@ -576,7 +613,7 @@ export default { .option('watch', { type: 'boolean', describe: 'Reload on changes; run --build again on changes under src/ and public/' }) .option('run', { type: 'string', describe: 'Long-running command (a dev server) to start beside the server' }) .option('proxy', { type: 'string', describe: 'Dev server URL for requests that no rule ends' }) - .option('pull-rules', { type: 'boolean', describe: 'Read portal-managed rules (a later release)' }) + .option('pull-rules', { type: 'boolean', describe: 'Read the portal rules at start and run them with the local rules (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) .option('kv-remote', { type: 'string', describe: 'Read-only remote KV store (a later release)' }) .option('version-info', { type: 'boolean', describe: 'Print the versions and exit' }), promptArgs: async () => { diff --git a/src/serve/portal/pull-rules.js b/src/serve/portal/pull-rules.js new file mode 100644 index 0000000..e473d84 --- /dev/null +++ b/src/serve/portal/pull-rules.js @@ -0,0 +1,174 @@ +/** + * `quant serve --pull-rules`: read the rules of the project from the portal + * one time at start, and run them locally in edge order with the rules of + * quant/edge-rules.json. + * + * The pulled list and its action settings stay in memory only. A rule's + * settings can hold credentials, so no message prints a setting value. + */ + +import { PortalError } from './client.js'; +import { orderRules } from '../rules.js'; + +export const PARTIAL_WARNING = 'Some rule types are not in the portal response for this organisation, so they do not run locally.'; +export const HEADERS_SCOPE_WARNING = 'QUANT_API_TOKEN lacks projects:read, so the portal\'s global headers are not used.'; + +// Action settings that are booleans, but can arrive as text. +const BOOLEAN_SETTINGS = [ + 'only_proxy_404', 'static_error_page', 'waf_enabled', 'disable_ssl_verify', 'failover_mode', + 'proxy_alert_enabled', 'proxy_inline_fn_enabled', 'application_proxy' +]; +// Action settings that are lists of status codes. +const STATUS_LISTS = ['status_codes', 'static_error_page_status_codes']; +// Request conditions that quant serve does not check. A rule that uses one is skipped. +const UNCHECKED_CONDITIONS = ['country', 'ip', 'asn', 'ua', 'traffic_source']; +const TRUE_TEXT = ['true', '1']; + +const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const projectPath = ({ customer, project }, leaf) => + `/organizations/${encodeURIComponent(customer)}/projects/${encodeURIComponent(project)}/${leaf}`; + +function toBoolean(value) { + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return value === 1; + return typeof value === 'string' && TRUE_TEXT.includes(value.trim().toLowerCase()); +} + +function toList(value) { + if (Array.isArray(value)) return value; + return ['string', 'number'].includes(typeof value) ? [value] : []; +} + +const textList = (value) => toList(value).filter((entry) => typeof entry === 'string'); + +const validItem = (entry) => isObject(entry) && Number.isInteger(entry.position) && entry.position >= 0; + +/** + * Read the rules list of the project, in edge order. + * @returns {Promise<{ rules: Object[], partial: boolean }>} partial: the + * positions are not exactly 0 to n-1, so some rule types are not in the list. + */ +export async function pullRules(client, ids) { + const { status, body } = await client.get(projectPath(ids, 'rules'), undefined, { scope: 'rules:read', what: 'portal rules' }); + if (status === 404) { + throw new PortalError(`The portal did not find project ${ids.project} in organisation ${ids.customer} (404).`, 404); + } + if (!Array.isArray(body) || !body.every(validItem)) throw new PortalError('The portal sent a rules list that is not valid.', status); + return { rules: body, partial: body.some((entry, index) => entry.position !== index) }; +} + +function normalConfig(value) { + const config = isObject(value) ? { ...value } : {}; + for (const name of BOOLEAN_SETTINGS.filter((key) => key in config)) config[name] = toBoolean(config[name]); + for (const name of STATUS_LISTS.filter((key) => key in config)) config[name] = toList(config[name]); + return config; +} + +// The allowed and the refused methods of the method condition. +function methodLists(item) { + const upper = (value) => textList(value).map((method) => method.toUpperCase()); + if (item.method === 'method_is') return { methods: upper(item.method_is), methodsNot: null }; + if (item.method === 'method_is_not') return { methods: null, methodsNot: upper(item.method_is_not) }; + return { methods: null, methodsNot: null }; +} + +const uncheckedConditions = (item) => UNCHECKED_CONDITIONS.filter((name) => typeof item[name] === 'string' && item[name] !== 'any'); + +function ruleId(item) { + return String(item.uuid ?? item.rule_id ?? `position-${item.position}`); +} + +function domains(value) { + const list = textList(value).map((domain) => domain.toLowerCase()); + return list.length > 0 ? list : ['any']; +} + +/** The portal list item as the local runtime runs it. */ +export function fromPortalRule(item) { + const config = normalConfig(item.action_config); + const action = String(item.action ?? 'rule'); + const unchecked = uncheckedConditions(item); + const weight = Number(item.weight); + return { + id: ruleId(item), + name: typeof item.name === 'string' && item.name !== '' ? item.name : null, + type: action, + action, + config, + url: textList(item.url), + domain: domains(item.domain), + ...methodLists(item), + weight: Number.isFinite(weight) ? weight : 0, + disabled: toBoolean(item.disabled) || unchecked.length > 0, + fnUuid: typeof config.fn_uuid === 'string' ? config.fn_uuid.toLowerCase() : null, + managed: item.managed === true, + unchecked + }; +} + +/** + * The rules that run: the pulled list in its order without the rules that a + * template deploy owns (managed), and each local rule (in weight order) + * before the first pulled rule with a strictly higher weight. A local rule + * with a weight higher than all goes last. + */ +export function mergeRules(pulled, localManaged) { + const merged = pulled.filter((rule) => !rule.managed); + const fromPortal = new Set(merged); + for (const rule of orderRules(localManaged)) { + const at = merged.findIndex((other) => fromPortal.has(other) && other.weight > rule.weight); + merged.splice(at === -1 ? merged.length : at, 0, rule); + } + return merged; +} + +function headersBody(body) { + if (Array.isArray(body) && body.length === 0) return {}; + if (!isObject(body)) throw new PortalError('The portal sent project headers that are not valid.'); + return body; +} + +/** + * Read the global headers of the project. A token without projects:read + * gives a warning and null. + * @returns {Promise|null>} + */ +export async function pullGlobalHeaders(client, { customer, project, warn = () => {} }) { + try { + const { status, body } = await client.get(projectPath({ customer, project }, 'custom-headers'), undefined, { scope: 'projects:read', what: 'project headers' }); + return status === 404 ? null : headersBody(body); + } catch (err) { + if (!(err instanceof PortalError) || err.status !== 403) throw err; + warn(HEADERS_SCOPE_WARNING); + return null; + } +} + +const ruleLabel = (rule) => rule.name || rule.id; + +function conditionWarning(rule) { + const names = rule.unchecked.join(', '); + const noun = rule.unchecked.length === 1 ? 'condition is' : 'conditions are'; + return `Rule ${ruleLabel(rule)}: the ${names} ${noun} not checked in quant serve, so the rule is skipped.`; +} + +/** + * Pull the rules and the global headers of the project. + * @returns {Promise<{ rules: Object[], headers: Record|null, count: number, managed: number, warnings: string[] }>} + * rules: runtime rules in edge order (with the managed rules); count: the pulled rules; + * managed: the pulled rules that come from quant/edge-rules.json. + */ +export async function pullProject(client, { customer, project }) { + const pulled = await pullRules(client, { customer, project }); + const rules = pulled.rules.map(fromPortalRule); + const warnings = pulled.partial ? [PARTIAL_WARNING] : []; + const headers = await pullGlobalHeaders(client, { customer, project, warn: (line) => warnings.push(line) }); + const skipped = rules.filter((rule) => !rule.managed && rule.unchecked.length > 0); + return { + rules, + headers, + count: rules.length, + managed: rules.filter((rule) => rule.managed).length, + warnings: [...warnings, ...skipped.map(conditionWarning)] + }; +} diff --git a/src/serve/project-config.js b/src/serve/project-config.js index 3adf43e..bcacbe2 100644 --- a/src/serve/project-config.js +++ b/src/serve/project-config.js @@ -128,6 +128,37 @@ export function readServeConfig(root, source = SERVE_CONFIG) { return parseServeConfig(text === null ? {} : parseJson(text, source), source); } +// The pulled headers that are valid, in lowercase. A header that is not +// valid is skipped with a warning that does not print its value. +function pulledEntries(pulled, warnings) { + const entries = []; + for (const [name, raw] of Object.entries(pulled)) { + const value = raw === null || raw === undefined ? '' : String(raw); + const lower = name.toLowerCase(); + if (!HEADER_NAME.test(name) || !HEADER_VALUE.test(value)) warnings.push('A portal global header is not valid and is not used.'); + else if (PROTECTED.includes(lower)) warnings.push(`The portal global header ${lower} is not applied. A project cannot set the ${lower} header.`); + else entries.push([lower, value]); + } + return entries; +} + +/** + * The settings with the pulled (portal) global headers as the base. A + * globalHeaders name of quant/serve.json overrides the pulled header of the + * same name, with one warning for each name. Null `pulled` changes nothing. + */ +export function withPulledHeaders(settings, pulled, source = SERVE_CONFIG) { + if (!pulled) return settings; + const local = new Map(settings.globalHeaders); + const warnings = []; + const base = pulledEntries(pulled, warnings).filter(([name]) => { + if (!local.has(name)) return true; + warnings.push(`${source} globalHeaders.${name} replaces the portal global header of the same name.`); + return false; + }); + return { ...settings, globalHeaders: [...base, ...local], warnings: [...settings.warnings, ...warnings] }; +} + /** The global headers as an object, for a function response to start from. */ export function globalHeaderObject(settings) { return Object.fromEntries(settings.globalHeaders); diff --git a/src/serve/project.js b/src/serve/project.js index 2a17cd7..bbfd53e 100644 --- a/src/serve/project.js +++ b/src/serve/project.js @@ -11,7 +11,8 @@ import { assertManifest } from '../helper/functions-manifest.js'; import { resolveRules, ruleErrors, toRuntimeRule } from '../helper/edge-rules-manifest.js'; import { parseMeta } from '../helper/quant-meta.js'; import { readInlineSetting } from './inline.js'; -import { readServeConfig, SERVE_CONFIG } from './project-config.js'; +import { readServeConfig, SERVE_CONFIG, withPulledHeaders } from './project-config.js'; +import { mergeRules } from './portal/pull-rules.js'; import { actionWarnings } from './rule-actions.js'; import { orderRules } from './rules.js'; @@ -92,18 +93,21 @@ function ruleWarning(rule, fn, source) { * Load and check the project files. A missing file counts as empty. * @param {Object} options * @param {string} options.root The project directory. File paths resolve from it. - * @param {Object[]} [options.runtimeRules] Runtime rules (with action and config) that run with - * the rules of the rules manifest, ordered by weight. + * @param {Object[]|null} [options.pulledRules] --pull-rules: the portal rules in edge order. The + * rules of the rules manifest replace the managed ones (see mergeRules). Kept in memory only. + * @param {Record|null} [options.pulledHeaders] --pull-rules: the portal global + * headers. The globalHeaders of quant/serve.json override them. * @returns {{ functions: Map, rules: Object[], stores: Object[], fixtures: string, inlineFunctions: boolean, warnings: string[] }} * functions is keyed by lowercase uuid; rules are runtime rules in run order. * inlineFunctions: project_settings.enable_inline_fn in meta.json. * settings: the local project settings from serve.json (see project-config.js). */ -export function loadProject({ root, functions = DEFAULT_FILES.functions, rules = DEFAULT_FILES.rules, meta = DEFAULT_FILES.meta, fixtures = DEFAULT_FILES.fixtures, serve = DEFAULT_FILES.serve, runtimeRules: extraRules = [] }) { +export function loadProject({ root, functions = DEFAULT_FILES.functions, rules = DEFAULT_FILES.rules, meta = DEFAULT_FILES.meta, fixtures = DEFAULT_FILES.fixtures, serve = DEFAULT_FILES.serve, pulledRules = null, pulledHeaders = null }) { const manifest = readFunctions(root, functions); - const runtimeRules = orderRules([...readRules(root, rules, manifest), ...extraRules]); + const localRules = readRules(root, rules, manifest); + const runtimeRules = pulledRules ? mergeRules(pulledRules, localRules) : localRules; const map = functionMap(root, manifest); - const settings = readServeConfig(root, serve); + const settings = withPulledHeaders(readServeConfig(root, serve), pulledHeaders, serve); const warnings = [ ...runtimeRules.map((rule) => ruleWarning(rule, map.get(rule.fnUuid), functions)).filter(Boolean), ...actionWarnings(runtimeRules) diff --git a/src/serve/rule-actions.js b/src/serve/rule-actions.js index 109feb7..5538c6d 100644 --- a/src/serve/rule-actions.js +++ b/src/serve/rule-actions.js @@ -13,6 +13,8 @@ import { ruleCaptures } from './rules.js'; const SUPPORTED = new Set(['edge_fn', 'edge_auth', 'edge_filter', 'redirect', 'headers', 'custom_response', 'auth', 'error_page', 'proxy']); // Headers that a headers rule cannot set. const PROTECTED = new Set(['quant-server', 'age', 'quant-project', 'quant-url-hash', 'quant-orig-host']); +// Headers that a proxy rule cannot inject: hop-by-hop and length headers. +const NOT_INJECTED = new Set(['content-length', 'transfer-encoding', 'connection', 'keep-alive', 'upgrade', 'te', 'trailer', 'host']); const REDIRECT_CODES = [301, 302, 303, 307, 308]; const NULL_BODY = [204, 205, 304]; const DEFAULT_ORIGIN_TIMEOUT_MS = 30000; @@ -130,7 +132,9 @@ function proxyTarget(rule, ctx) { // setting is not used (start-up warns). function originRequestHeaders(ctx, config) { const headers = withoutHeaders(proxyRequestHeaders(ctx.req), nameList(config.proxy_strip_request_headers)); - for (const [name, value] of headerEntries(config.inject_headers)) headers.set(name, value); + for (const [name, value] of headerEntries(config.inject_headers)) { + if (!NOT_INJECTED.has(name)) headers.set(name, value); + } if (config.auth_user) headers.set('authorization', basicCredentials(config.auth_user, config.auth_pass)); return headers; } @@ -228,11 +232,21 @@ export async function runAction(state, ctx, rule) { return action ? action(state, ctx, rule) : null; } +// A setting that is on: true, a number, or text or a list that is not empty. +const isOn = (value) => value === true || typeof value === 'number' || (typeof value === 'string' && value !== '') || + (Array.isArray(value) && value.length > 0); + +// TLS verification and failover work differently on this machine. Notification, +// alert and application settings have no local effect and get no warning. +const changesOriginTrust = (config) => config.disable_ssl_verify === true || + Object.keys(config).some((name) => name.startsWith('failover_') && isOn(config[name])); + function proxyWarnings(rule) { const config = configOf(rule); return [ ...(config.waf_enabled ? [`Rule ${ruleLabel(rule)}: WAF settings are ignored in quant serve.`] : []), - ...(config.host ? [`Rule ${ruleLabel(rule)}: the host setting is ignored in quant serve. The origin gets the host of the to URL.`] : []) + ...(config.host ? [`Rule ${ruleLabel(rule)}: the host setting is ignored in quant serve. The origin gets the host of the to URL.`] : []), + ...(changesOriginTrust(config) ? [`Rule ${ruleLabel(rule)}: failover and TLS verification settings work differently in quant serve.`] : []) ]; } diff --git a/src/serve/rules.js b/src/serve/rules.js index 4ec18b4..ea4535f 100644 --- a/src/serve/rules.js +++ b/src/serve/rules.js @@ -49,9 +49,14 @@ function hostMatches(domains, host) { return domains.some(domain => domain === 'any' || domain === bare); } +function methodAllowed(rule, method) { + if (rule.methods && !rule.methods.includes(method)) return false; + return !(rule.methodsNot && rule.methodsNot.includes(method)); +} + function ruleMatches(rule, ctx) { if (rule.disabled) return false; - if (rule.methods && !rule.methods.includes(ctx.method)) return false; + if (!methodAllowed(rule, ctx.method)) return false; if (!hostMatches(rule.domain, ctx.host)) return false; return rule.url.some(pattern => urlMatches(pattern, ctx.pathname, ctx.search || '')); } @@ -78,7 +83,11 @@ export function matchFilter(rules, ctx) { return rules.find(rule => rule.action === 'edge_filter' && ruleMatches(rule, ctx)) || null; } -const listsStatus = (rule, status) => (rule.config?.status_codes || []).map(String).includes(String(status)); +// A status_codes value that is not a list lists no status. +function listsStatus(rule, status) { + const codes = rule.config?.status_codes; + return Array.isArray(codes) && codes.map(String).includes(String(status)); +} /** The first enabled error page rule for the status that matches, or null. */ export function matchErrorPage(rules, ctx, status) { diff --git a/src/serve/server.js b/src/serve/server.js index c8f4684..f273062 100644 --- a/src/serve/server.js +++ b/src/serve/server.js @@ -565,8 +565,8 @@ async function handleUpgrade(state, req, socket, head) { // --- Start-up ------------------------------------------------------------------- function projectFiles(state) { - const { functions, rules, meta, fixtures, runtimeRules } = state.options; - return { root: state.root, functions, rules, meta, fixtures, runtimeRules }; + const { functions, rules, meta, fixtures, pulledRules, pulledHeaders } = state.options; + return { root: state.root, functions, rules, meta, fixtures, pulledRules, pulledHeaders }; } function siteIndex(state) { @@ -710,7 +710,8 @@ function closer(state, server, cleanups) { * root, dir, port, host, functions, rules, meta, fixtures, org, project, * domain, proxy, watch, geo, clientIp, timeoutMs, kvStore, log, onSourceChange, * backend (the local KV backend, for the control route), clock ('real' or 'manual') - * and runtimeRules (runtime rules that run with the rules of the rules manifest, by weight). + * pulledRules (--pull-rules: the portal rules in edge order, merged with the rules manifest) + * and pulledHeaders (--pull-rules: the portal global headers, or null). * @returns {Promise<{ url: string, close: () => Promise, setRunPid: (pid: number) => void, summary: Object }>} * setRunPid: the --run process group that the watchdog ends when a function blocks the server. * summary: { stores, rules, libraries, warnings } for the start-up output. diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 08b45f5..24ed041 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -197,7 +197,7 @@ describe('serve command', () => { expect(() => resolveOptions({ clock: 'fast' }, { root: tmp, gitUserName: noGit })).to.throw('--clock must be real or manual'); }); - for (const flag of [['--pull-rules'], ['--kv-remote', 'orbit']]) { + for (const flag of [['--kv-remote', 'orbit']]) { it(`stops for ${flag[0]}, which is available in a later release`, () => { expect(() => resolveOptions(parse(flag), { root: tmp, gitUserName: noGit })) .to.throw(`${flag[0]} is available in a later release`); diff --git a/tests/unit/serve/pull-rules.test.mjs b/tests/unit/serve/pull-rules.test.mjs new file mode 100644 index 0000000..df628fa --- /dev/null +++ b/tests/unit/serve/pull-rules.test.mjs @@ -0,0 +1,372 @@ +import { expect } from 'chai'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createPortalClient, PortalError } from '../../../src/serve/portal/client.js'; +import { + fromPortalRule, mergeRules, pullGlobalHeaders, pullProject, pullRules, + HEADERS_SCOPE_WARNING, PARTIAL_WARNING +} from '../../../src/serve/portal/pull-rules.js'; +import { loadProject } from '../../../src/serve/project.js'; +import { resolveOptions, runServe } from '../../../src/commands/serve.js'; + +const IDS = { customer: 'acme', project: 'site' }; +const TOKEN = 'portal-token-never-printed'; +const BASE = new URL('http://127.0.0.1:8080/api/v2'); + +const reply = (status, body) => ({ status, headers: { get: () => null }, json: async () => body }); + +// A portal client on a fake fetch. `routes` maps a path suffix to a reply. +function fakePortal(routes) { + const calls = []; + const fetchImpl = async (url) => { + calls.push(String(url)); + const found = Object.entries(routes).find(([suffix]) => new URL(url).pathname.endsWith(suffix)); + return found ? found[1] : reply(404); + }; + return { calls, client: createPortalClient({ base: BASE, token: TOKEN, fetchImpl, sleep: async () => {} }) }; +} + +// A portal list item in the shape of the per-type lists. +function item(position, action, actionConfig = {}, over = {}) { + return { + uuid: `rule-${position}`, rule_id: `rule-${position}`, name: `Rule ${position}`, action, action_config: actionConfig, + url: ['/a/*'], domain: ['any'], disabled: false, weight: 0, position, managed: false, + country: 'any', method: 'any', ip: 'any', asn: 'any', ua: 'any', traffic_source: 'any', + ...over + }; +} + +// A runtime rule with an id and weight, for the merge tests. +const runtime = (id, weight, managed = false) => ({ id, weight, managed, url: ['/'], domain: ['any'], action: 'redirect', config: {} }); + +async function failure(promise) { + try { + await promise; + } catch (error) { + return error; + } + throw new Error('expected a rejection'); +} + +describe('serve --pull-rules', () => { + describe('pullRules', () => { + it('reads the rules list one time, with the rules:read scope', async () => { + const list = [item(0, 'redirect'), item(1, 'headers')]; + const { calls, client } = fakePortal({ '/organizations/acme/projects/site/rules': reply(200, list) }); + const pulled = await pullRules(client, IDS); + expect(calls).to.have.length(1); + expect(pulled.rules).to.deep.equal(list); + expect(pulled.partial).to.equal(false); + }); + + it('marks a list with gaps in its positions as partial', async () => { + const { client } = fakePortal({ '/rules': reply(200, [item(0, 'redirect'), item(2, 'headers')]) }); + expect((await pullRules(client, IDS)).partial).to.equal(true); + }); + + it('refuses a response that is not a list of rules with positions', async () => { + for (const body of [{}, [null], [{ action: 'redirect' }], [{ position: -1 }]]) { + const { client } = fakePortal({ '/rules': reply(200, body) }); + const error = await failure(pullRules(client, IDS)); + expect(error.message).to.equal('The portal sent a rules list that is not valid.'); + } + }); + + it('names the rules:read scope when the portal answers 403', async () => { + const { client } = fakePortal({ '/rules': reply(403, { message: 'no' }) }); + const error = await failure(pullRules(client, IDS)); + expect(error).to.be.instanceOf(PortalError); + expect(error.message).to.equal('QUANT_API_TOKEN lacks the rules:read scope for portal rules.'); + }); + + it('names the project when the portal answers 404', async () => { + const { client } = fakePortal({}); + const error = await failure(pullRules(client, IDS)); + expect(error.message).to.include('site').and.to.include('acme').and.to.include('404'); + }); + }); + + describe('fromPortalRule', () => { + const cases = { + redirect: { to: '/new', status_code: '302' }, + headers: { headers: { 'x-one': '1' } }, + custom_response: { custom_response_status_code: '200', custom_response_body: 'ok' }, + auth: { auth_user: 'user', auth_pass: 'pass' }, + error_page: { error_page_path: '/404.html', status_codes: ['404'] }, + proxy: { to: 'https://origin.test', only_proxy_404: false }, + edge_fn: { fn_uuid: 'AAAA0000-0000-4000-8000-000000000001' }, + edge_auth: { fn_uuid: 'aaaa0000-0000-4000-8000-000000000002' }, + edge_filter: { fn_uuid: 'aaaa0000-0000-4000-8000-000000000003' }, + robot_challenge: { robot_challenge_type: 'invisible' }, + serve_static_path: { static_file_path: '/a.html' } + }; + + for (const [action, config] of Object.entries(cases)) { + it(`maps a ${action} rule to the runtime shape`, () => { + const rule = fromPortalRule(item(3, action, config, { weight: 25 })); + expect(rule).to.include({ id: 'rule-3', name: 'Rule 3', action, type: action, weight: 25, disabled: false, managed: false }); + expect(rule.url).to.deep.equal(['/a/*']); + expect(rule.domain).to.deep.equal(['any']); + expect(rule.methods).to.equal(null); + expect(rule.config).to.deep.include(action === 'edge_fn' ? {} : config); + }); + } + + it('takes the function uuid of a function rule, in lowercase', () => { + expect(fromPortalRule(item(0, 'edge_fn', cases.edge_fn)).fnUuid).to.equal('aaaa0000-0000-4000-8000-000000000001'); + expect(fromPortalRule(item(0, 'redirect', cases.redirect)).fnUuid).to.equal(null); + }); + + it('makes the url and domain lists, and the domains lowercase', () => { + const rule = fromPortalRule(item(0, 'redirect', {}, { url: '/one', domain: ['Example.COM'] })); + expect(rule.url).to.deep.equal(['/one']); + expect(rule.domain).to.deep.equal(['example.com']); + expect(fromPortalRule(item(0, 'redirect', {}, { domain: [] })).domain).to.deep.equal(['any']); + }); + + it('reads the method condition', () => { + const only = fromPortalRule(item(0, 'redirect', {}, { method: 'method_is', method_is: ['get', 'POST'] })); + expect(only.methods).to.deep.equal(['GET', 'POST']); + const not = fromPortalRule(item(0, 'redirect', {}, { method: 'method_is_not', method_is_not: ['DELETE'] })); + expect(not.methods).to.equal(null); + expect(not.methodsNot).to.deep.equal(['DELETE']); + }); + + it('skips a rule with a condition that quant serve does not check', () => { + const rule = fromPortalRule(item(0, 'redirect', {}, { country: 'country_is', country_is: ['AU'] })); + expect(rule.disabled).to.equal(true); + expect(rule.unchecked).to.deep.equal(['country']); + expect(fromPortalRule(item(0, 'redirect')).unchecked).to.deep.equal([]); + }); + + it('makes booleans of boolean settings sent as text', () => { + const rule = fromPortalRule(item(0, 'proxy', { + only_proxy_404: 'true', static_error_page: '', waf_enabled: '0', disable_ssl_verify: '1', failover_mode: 'false' + }, { disabled: 'true' })); + expect(rule.disabled).to.equal(true); + expect(rule.config).to.include({ + only_proxy_404: true, static_error_page: false, waf_enabled: false, disable_ssl_verify: true, failover_mode: false + }); + expect(fromPortalRule(item(0, 'redirect', {}, { disabled: '0' })).disabled).to.equal(false); + }); + + it('makes status_codes a list', () => { + expect(fromPortalRule(item(0, 'error_page', { status_codes: '404' })).config.status_codes).to.deep.equal(['404']); + expect(fromPortalRule(item(0, 'error_page', { status_codes: 500 })).config.status_codes).to.deep.equal([500]); + expect(fromPortalRule(item(0, 'error_page', { status_codes: { a: 1 } })).config.status_codes).to.deep.equal([]); + expect(fromPortalRule(item(0, 'error_page', { status_codes: null })).config.status_codes).to.deep.equal([]); + }); + + it('marks a rule that a template deploy owns', () => { + expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: true })).managed).to.equal(true); + }); + }); + + describe('mergeRules', () => { + const ids = (rules) => rules.map((rule) => rule.id); + const pulled = [runtime('A', 0), runtime('B', 100), runtime('C', 200)]; + + it('puts a local rule before the first pulled rule with a higher weight', () => { + expect(ids(mergeRules(pulled, [runtime('M', 150)]))).to.deep.equal(['A', 'B', 'M', 'C']); + }); + + it('keeps a pulled rule of the same weight first', () => { + expect(ids(mergeRules(pulled, [runtime('M', 100)]))).to.deep.equal(['A', 'B', 'M', 'C']); + }); + + it('puts a local rule with a negative weight first', () => { + expect(ids(mergeRules(pulled, [runtime('M', -5)]))).to.deep.equal(['M', 'A', 'B', 'C']); + }); + + it('puts a local rule with a weight higher than all last', () => { + expect(ids(mergeRules(pulled, [runtime('M', 500)]))).to.deep.equal(['A', 'B', 'C', 'M']); + }); + + it('keeps the pulled order and puts the local rules in weight order', () => { + const unsorted = [runtime('A', 300), runtime('B', 100)]; + expect(ids(mergeRules(unsorted, [runtime('N', 200), runtime('M', 50)]))).to.deep.equal(['M', 'N', 'A', 'B']); + expect(ids(mergeRules(unsorted, [runtime('M', 50), runtime('N', 50)]))).to.deep.equal(['M', 'N', 'A', 'B']); + }); + + it('replaces the pulled managed rules with the local rules', () => { + const withManaged = [runtime('A', 0), runtime('old', 150, true), runtime('C', 200)]; + expect(ids(mergeRules(withManaged, [runtime('M', 150)]))).to.deep.equal(['A', 'M', 'C']); + expect(ids(mergeRules(withManaged, []))).to.deep.equal(['A', 'C']); + }); + }); + + describe('pullGlobalHeaders', () => { + it('reads the project headers with the projects:read scope', async () => { + const { calls, client } = fakePortal({ '/organizations/acme/projects/site/custom-headers': reply(200, { 'X-Frame-Options': 'DENY' }) }); + expect(await pullGlobalHeaders(client, IDS)).to.deep.equal({ 'X-Frame-Options': 'DENY' }); + expect(calls).to.have.length(1); + }); + + it('warns and gives null when the token lacks projects:read', async () => { + const warnings = []; + const { client } = fakePortal({ '/custom-headers': reply(403, {}) }); + expect(await pullGlobalHeaders(client, { ...IDS, warn: (line) => warnings.push(line) })).to.equal(null); + expect(warnings).to.deep.equal([HEADERS_SCOPE_WARNING]); + expect(HEADERS_SCOPE_WARNING).to.equal('QUANT_API_TOKEN lacks projects:read, so the portal\'s global headers are not used.'); + }); + + it('takes an empty list as no headers', async () => { + const { client } = fakePortal({ '/custom-headers': reply(200, []) }); + expect(await pullGlobalHeaders(client, IDS)).to.deep.equal({}); + }); + }); + + describe('pullProject', () => { + it('gives the runtime rules, the counts, the headers and one partial warning', async () => { + const list = [item(0, 'redirect'), item(2, 'edge_fn', {}, { managed: true }), item(4, 'headers', {}, { ip: 'ip_is', ip_is: ['1.2.3.4'] })]; + const { calls, client } = fakePortal({ '/rules': reply(200, list), '/custom-headers': reply(200, { 'x-a': '1' }) }); + const pulled = await pullProject(client, IDS); + expect(calls).to.have.length(2); + expect(pulled).to.include({ count: 3, managed: 1 }); + expect(pulled.headers).to.deep.equal({ 'x-a': '1' }); + expect(pulled.rules.map((rule) => rule.id)).to.deep.equal(['rule-0', 'rule-2', 'rule-4']); + expect(pulled.warnings.filter((line) => line === PARTIAL_WARNING)).to.have.length(1); + expect(PARTIAL_WARNING).to.equal('Some rule types are not in the portal response for this organisation, so they do not run locally.'); + expect(pulled.warnings).to.include('Rule Rule 4: the ip condition is not checked in quant serve, so the rule is skipped.'); + }); + }); + + describe('global headers from a pull', () => { + let tmp; + beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-pull-headers-')); }); + afterEach(() => fs.rmSync(tmp, { recursive: true, force: true })); + + it('uses the pulled headers as the base and lets quant/serve.json override them', () => { + fs.mkdirSync(path.join(tmp, 'quant')); + fs.writeFileSync(path.join(tmp, 'quant', 'serve.json'), JSON.stringify({ globalHeaders: { 'X-Shared': 'local' } })); + const project = loadProject({ root: tmp, pulledHeaders: { 'x-shared': 'pv-shared', 'X-Portal': 'pv-portal', age: '9' } }); + expect(new Map(project.settings.globalHeaders)).to.deep.equal(new Map([['x-portal', 'pv-portal'], ['x-shared', 'local']])); + expect(project.warnings).to.include('quant/serve.json globalHeaders.x-shared replaces the portal global header of the same name.'); + expect(project.warnings.join('\n')).not.to.include('pv-'); + }); + + it('keeps the local headers only when there is no pull', () => { + const project = loadProject({ root: tmp }); + expect(project.settings.globalHeaders).to.deep.equal([]); + }); + }); + + describe('runServe', function () { + this.timeout(15000); + const PASSWORD = 'pulled-password-never-printed'; + const SECRET = 'pulled-secret-never-printed'; + const HEADER_SECRET = 'pulled-header-never-printed'; + const FN = 'aaaa0000-0000-4000-8000-0000000000aa'; + let tmp; + + beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-pull-serve-')); }); + afterEach(() => fs.rmSync(tmp, { recursive: true, force: true })); + + function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(value, null, 2)); + } + + function localRules(url) { + return { version: 1, namespace: 'test', rules: [{ id: 'local-m', name: 'Local M', type: 'function', urls: [url], weight: 150, function_uuid: FN }] }; + } + + function projectRoot() { + const root = path.join(tmp, 'site'); + fs.mkdirSync(path.join(root, 'dist'), { recursive: true }); + fs.writeFileSync(path.join(root, 'dist', 'index.html'), 'home'); + fs.mkdirSync(path.join(root, 'quant', 'edge-functions'), { recursive: true }); + fs.writeFileSync(path.join(root, 'quant', 'edge-functions', 'm.js'), 'return new Response(\'local M\');'); + writeJson(path.join(root, 'quant', 'edge-functions.json'), [{ id: 'm', uuid: FN, type: 'function', path: 'quant/edge-functions/m.js', description: 'm' }]); + writeJson(path.join(root, 'quant', 'edge-rules.json'), localRules('/m')); + return root; + } + + const LIST = [ + item(0, 'redirect', { to: '/moved', status_code: '302' }, { uuid: 'a', name: 'A', url: ['/a'], weight: 0 }), + item(1, 'auth', { auth_user: 'ada', auth_pass: PASSWORD }, { uuid: 'b', name: 'B', url: ['/staff'], weight: 100 }), + item(2, 'edge_fn', { fn_uuid: FN }, { uuid: 'old', name: 'Old', url: ['/m'], weight: 150, managed: true }), + item(4, 'proxy', { to: 'http://127.0.0.1:1$*', inject_headers: { 'x-shared': SECRET }, disable_ssl_verify: 'true' }, { uuid: 'p', name: 'P', url: ['/api/*'], weight: 160 }), + item(5, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'pulled C' }, { uuid: 'c', name: 'C', url: ['/m', '/n'], weight: 200 }) + ]; + + function portalFetch(calls) { + return async (url, init) => { + calls.push({ url: String(url), auth: init.headers.Authorization }); + const { pathname } = new URL(url); + if (pathname.endsWith('/organizations/acme/projects/site/rules')) return reply(200, LIST); + if (pathname.endsWith('/organizations/acme/projects/site/custom-headers')) return reply(200, { 'x-portal': HEADER_SECRET }); + return reply(404); + }; + } + + function filesUnder(dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + return entry.isDirectory() ? filesUnder(full) : [full]; + }); + } + + async function until(check, timeoutMs = 8000) { + const started = Date.now(); + while (!(await check())) { + if (Date.now() - started > timeoutMs) throw new Error('timed out'); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + + it('runs the pulled rules in edge order with the local rules, and keeps them in memory only', async () => { + const root = projectRoot(); + const calls = []; + const output = []; + let stop; + const stopped = new Promise((resolve) => { stop = resolve; }); + let url; + const opts = resolveOptions({ dir: 'dist', port: 0, 'pull-rules': true, org: 'acme', project: 'site', watch: true }, { root, gitUserName: () => null }); + const running = runServe(opts, { + stop: stopped, + print: (line) => output.push(line), + log: (line) => output.push(line), + env: { QUANT_API_TOKEN: TOKEN, QUANT_BASE_URL: 'http://127.0.0.1:8080/api/v2' }, + fetchImpl: portalFetch(calls), + onListening: (server) => { url = server.url; } + }); + await until(() => url); + try { + expect(calls).to.have.length(2); + expect(output).to.include('Pulled 5 portal rules (1 from quant/edge-rules.json).'); + expect(output.filter((line) => line.includes(PARTIAL_WARNING))).to.have.length(1); + expect(output.join('\n')).to.include('Rule P: failover and TLS verification settings work differently in quant serve.'); + expect((await fetch(`${url}/a`, { redirect: 'manual' })).headers.get('location')).to.equal('/moved'); + expect((await fetch(`${url}/staff`)).status).to.equal(401); + expect(await (await fetch(`${url}/m`)).text()).to.equal('local M'); + expect(await (await fetch(`${url}/n`)).text()).to.equal('pulled C'); + expect((await fetch(`${url}/`)).headers.get('x-portal')).to.equal(HEADER_SECRET); + + // --watch: a change to quant/edge-rules.json merges again, with no new pull. + await new Promise((resolve) => setTimeout(resolve, 300)); + writeJson(path.join(root, 'quant', 'edge-rules.json'), localRules('/n')); + await until(async () => (await (await fetch(`${url}/n`)).text()) === 'local M'); + expect(await (await fetch(`${url}/m`)).text()).to.equal('pulled C'); + expect(calls).to.have.length(2); + } finally { + stop(); + await running; + } + const printed = output.join('\n'); + for (const secret of [TOKEN, PASSWORD, SECRET, HEADER_SECRET]) { + expect(printed, secret).to.not.include(secret); + for (const file of filesUnder(root)) expect(fs.readFileSync(file, 'latin1'), `${secret} in ${file}`).to.not.include(secret); + } + expect(fs.existsSync(path.join(root, '.quant'))).to.equal(true); + }); + + it('needs a named organisation and project', () => { + const root = projectRoot(); + expect(() => resolveOptions({ 'pull-rules': true }, { root, gitUserName: () => null })) + .to.throw('--pull-rules needs the portal organisation and project'); + expect(resolveOptions({ 'pull-rules': true, org: 'acme', project: 'site' }, { root, gitUserName: () => null }).pullRules).to.equal(true); + }); + }); +}); diff --git a/tests/unit/serve/rule-actions.test.mjs b/tests/unit/serve/rule-actions.test.mjs index 03199eb..d0dfef7 100644 --- a/tests/unit/serve/rule-actions.test.mjs +++ b/tests/unit/serve/rule-actions.test.mjs @@ -29,10 +29,10 @@ function project(files = {}) { return root; } -async function start(runtimeRules, files = {}) { +async function start(pulledRules, files = {}) { const server = await startServer({ root: project(files), dir: 'dist', port: 0, host: '127.0.0.1', org: 'acme', project: 'site', - runtimeRules, log: (line) => lines.push(line) + pulledRules, log: (line) => lines.push(line) }); servers.push(server); return server; @@ -275,6 +275,22 @@ describe('serve rule actions', () => { expect(seen['x-keep']).to.equal('1'); }); + it('does not inject hop-by-hop or length headers', async () => { + const up = await origin(echo); + const inject = { + 'content-length': '999', 'transfer-encoding': 'chunked', connection: 'x-keep', 'keep-alive': '5', + upgrade: 'websocket', te: 'trailers', trailer: 'x-t', host: 'evil.test', 'x-ok': 'yes' + }; + const server = await start([rule('proxy', ['*'], { to: `${up.url}$*`, inject_headers: inject })]); + const res = await get(server, '/p'); + expect(res.status).to.equal(201); + const seen = up.seen[0].headers; + expect(seen['x-ok']).to.equal('yes'); + expect(seen.host).to.equal(new URL(up.url).host); + for (const name of ['transfer-encoding', 'keep-alive', 'upgrade', 'te', 'trailer']) expect(seen[name], name).to.equal(undefined); + expect(seen['content-length']).to.not.equal('999'); + }); + it('sets cache-control from cache_lifetime', async () => { const up = await origin(echo); const server = await start([ @@ -352,6 +368,19 @@ describe('serve rule actions', () => { expect(up.seen[0].headers.host).to.equal(new URL(up.url).host); }); + it('warns once, with no value, about failover and TLS verification settings', () => { + const text = 'Rule API: failover and TLS verification settings work differently in quant serve.'; + const tls = rule('proxy', ['*'], { to: 'https://o.test', disable_ssl_verify: true }, { id: 'a', name: 'API' }); + const failover = rule('proxy', ['*'], { to: 'https://o.test', failover_mode: true, failover_origin_ttfb: '2000' }, { id: 'b', name: 'API' }); + const quiet = rule('proxy', ['*'], { + to: 'https://o.test', disable_ssl_verify: false, failover_mode: false, failover_lifetime: null, failover_origin_status_codes: [], + notify_config: { slack_webhook: 'https://hooks.test/x' }, proxy_alert_enabled: true, application_proxy: true, application_name: 'app' + }, { id: 'c', name: 'Quiet' }); + expect(actionWarnings([tls])).to.deep.equal([text]); + expect(actionWarnings([failover])).to.deep.equal([text]); + expect(actionWarnings([quiet])).to.deep.equal([]); + }); + it('gives one warning for each rule id', () => { const twice = rule('bot_challenge', ['*'], {}, { id: 'bc' }); expect(actionWarnings([twice, twice])).to.deep.equal(['Rule bot_challenge rule (bot_challenge) is not available in quant serve and is skipped.']); diff --git a/tests/unit/serve/rules.test.mjs b/tests/unit/serve/rules.test.mjs index 0d425c7..969ebff 100644 --- a/tests/unit/serve/rules.test.mjs +++ b/tests/unit/serve/rules.test.mjs @@ -67,6 +67,12 @@ describe('serve rules', () => { expect(matchRule([r], ctx({ method: 'POST' }))).to.equal(r); }); + it('refuses a method in the methodsNot list', () => { + const r = rule({ methodsNot: ['DELETE'] }); + expect(matchRule([r], ctx({ method: 'DELETE' }))).to.equal(null); + expect(matchRule([r], ctx({ method: 'GET' }))).to.equal(r); + }); + it('skips disabled rules', () => { expect(matchRule([rule({ disabled: true })], ctx())).to.equal(null); }); @@ -106,6 +112,12 @@ describe('serve rules', () => { expect(matchErrorPage([e], ctx(), 500)).to.equal(null); expect(matchErrorPage([page({ disabled: true })], ctx(), 404)).to.equal(null); }); + + it('do not throw when status_codes is not a list', () => { + for (const codes of ['404', 404, { a: 1 }, null]) { + expect(matchErrorPage([page({ config: { status_codes: codes } })], ctx(), 404)).to.equal(null); + } + }); }); describe('ruleCaptures', () => { From bc78b7e77be0ee3fdabbe7381e0d8c4743f0d4c4 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:25:00 +1000 Subject: [PATCH 17/32] fix(serve): pull-rules review fixes --- README.md | 5 ++- src/commands/serve.js | 9 ++++- src/serve/portal/pull-rules.js | 55 +++++++++++++++++++++----- src/serve/project.js | 2 + src/serve/rule-actions.js | 10 ++--- src/serve/rules.js | 6 +++ src/serve/server.js | 6 ++- tests/unit/serve/pull-rules.test.mjs | 53 +++++++++++++++++++++++-- tests/unit/serve/rule-actions.test.mjs | 6 ++- tests/unit/serve/rules.test.mjs | 10 +++++ 10 files changed, 134 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 90945a5..fa3e3ab 100644 --- a/README.md +++ b/README.md @@ -479,10 +479,11 @@ quant serve dist --pull-rules --org acme --project marketing-site - The token and the portal URL come only from `QUANT_API_TOKEN` and `QUANT_BASE_URL`. The organisation and the project come from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. `--pull-rules` stops if no organisation or project is set. - The token needs the `rules:read` scope. Without it, `quant serve` stops with a message that names the scope. - The rules that a deploy of `quant/edge-rules.json` owns on the portal are replaced by the rules of the local file. Each local rule goes before the first portal rule with a higher weight. This is the order that the edge has after a rules deploy of the same file. -- The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of portal rules that the local file replaces. +- The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules from `quant/edge-rules.json` that run with the portal rules. - With `--watch`, a change to `quant/edge-rules.json` merges the local rules again with the portal rules in memory. `quant serve` does not read the portal again. - A rule type that quant serve does not run (for example a bot challenge) is skipped, with one warning at start-up. A rule with a country, IP, ASN, user agent or traffic source condition is skipped, with one warning. -- A proxy rule with failover settings or with TLS verification off gets one warning: these settings work differently in quant serve. +- A proxy rule with failover mode on or with TLS verification off gets one warning: these settings work differently in quant serve. +- A rule with a cookie condition runs only when the request sends a cookie of exactly that name. - For some organisations the portal leaves some rule types out of the list. Then `quant serve` prints one warning, and those rules do not run locally. - With the `projects:read` scope, the global headers of the portal project are used too. The `globalHeaders` of `quant/serve.json` override a portal header of the same name, with one warning for each name. Without the scope, `quant serve` prints one warning and uses only the local headers. - Basic authentication and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. diff --git a/src/commands/serve.js b/src/commands/serve.js index 8b4e786..9b170c2 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -224,7 +224,7 @@ function ledgerLines(ledgers, clock) { } function pullLines(pull) { - return pull ? [`Pulled ${pull.count} portal rules (${pull.managed} from quant/edge-rules.json).`] : []; + return pull ? [`Pulled ${pull.count} portal rules (${pull.local} from quant/edge-rules.json).`] : []; } function clockLines(clock) { @@ -453,6 +453,11 @@ async function pulledProject(options, { env = process.env, fetchImpl } = {}) { return pullProject(client, { customer, project }); } +// The start-up summary of a pull: the pulled rules and the local rules merged in. +function pullSummary(pulled, server) { + return pulled ? { count: pulled.count, local: server.summary.localRules } : null; +} + function withPull(options, pulled) { return pulled ? { ...options, pulledRules: pulled.rules, pulledHeaders: pulled.headers } : options; } @@ -528,7 +533,7 @@ export async function runServe(options, { stop, print = console.log, log, kvStor const server = await startOrShowKeys(serverOptions(withPull(options, pulled), { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); const pullWarnings = pulled ? pulled.warnings : []; const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...pullWarnings, ...server.summary.warnings] }; - const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, pull: pulled, ...startupSummary(summary, kv) }); + const lines = startupLines({ url: server.url, host: options.host, clock: options.clock, pull: pullSummary(pulled, server), ...startupSummary(summary, kv) }); for (const line of lines) print(line); const child = options.run ? startRun(options.run, options.root, print, options.proxy) : null; watchRun(server, child); diff --git a/src/serve/portal/pull-rules.js b/src/serve/portal/pull-rules.js index e473d84..cc8ed92 100644 --- a/src/serve/portal/pull-rules.js +++ b/src/serve/portal/pull-rules.js @@ -23,6 +23,10 @@ const STATUS_LISTS = ['status_codes', 'static_error_page_status_codes']; // Request conditions that quant serve does not check. A rule that uses one is skipped. const UNCHECKED_CONDITIONS = ['country', 'ip', 'asn', 'ua', 'traffic_source']; const TRUE_TEXT = ['true', '1']; +// A numeric string as the portal reads one: optional white space and sign, +// digits with an optional fraction, an optional exponent. +const NUMERIC_TEXT = /^[ \t\n\r\v\f]*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?[ \t\n\r\v\f]*$/; +const INT_LIMIT = 2 ** 63; const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); const projectPath = ({ customer, project }, leaf) => @@ -64,15 +68,48 @@ function normalConfig(value) { return config; } +const lower = (value) => (typeof value === 'string' ? value.trim().toLowerCase() : ''); + +// The uppercase methods of a method list; null (all methods) when it is empty. +function methodList(value) { + const list = textList(value).map((method) => method.toUpperCase()); + return list.length > 0 ? list : null; +} + // The allowed and the refused methods of the method condition. function methodLists(item) { - const upper = (value) => textList(value).map((method) => method.toUpperCase()); - if (item.method === 'method_is') return { methods: upper(item.method_is), methodsNot: null }; - if (item.method === 'method_is_not') return { methods: null, methodsNot: upper(item.method_is_not) }; + const mode = lower(item.method); + if (mode === 'method_is') return { methods: methodList(item.method_is), methodsNot: null }; + if (mode === 'method_is_not') return { methods: null, methodsNot: methodList(item.method_is_not) }; return { methods: null, methodsNot: null }; } -const uncheckedConditions = (item) => UNCHECKED_CONDITIONS.filter((name) => typeof item[name] === 'string' && item[name] !== 'any'); +// A condition is set unless it is empty or `any`, or its list is empty. +function conditionSet(item, name) { + const mode = lower(item[name]); + if (mode === '' || mode === 'any') return false; + const listed = [`${name}_is`, `${name}_is_not`].includes(mode); + return !listed || toList(item[mode]).length > 0; +} + +const uncheckedConditions = (item) => UNCHECKED_CONDITIONS.filter((name) => conditionSet(item, name)); + +const integral = (value) => Number.isFinite(value) && Number.isInteger(value) && value >= -INT_LIMIT && value < INT_LIMIT; + +/** + * The weight that the portal sorts by: an integral number or a numeric + * string with an integral value. Anything else (missing, a fraction, a + * boolean, other text, out of range) is 0. + */ +export function effectiveWeight(value) { + const number = typeof value === 'string' && NUMERIC_TEXT.test(value) ? Number(value.trim()) : value; + return typeof number === 'number' && integral(number) ? number : 0; +} + +// The cookie name of the cookie condition, or null for no condition. +function cookieName(value) { + return typeof value === 'string' && value.trim() !== '' ? value.trim() : null; +} function ruleId(item) { return String(item.uuid ?? item.rule_id ?? `position-${item.position}`); @@ -88,7 +125,6 @@ export function fromPortalRule(item) { const config = normalConfig(item.action_config); const action = String(item.action ?? 'rule'); const unchecked = uncheckedConditions(item); - const weight = Number(item.weight); return { id: ruleId(item), name: typeof item.name === 'string' && item.name !== '' ? item.name : null, @@ -98,7 +134,8 @@ export function fromPortalRule(item) { url: textList(item.url), domain: domains(item.domain), ...methodLists(item), - weight: Number.isFinite(weight) ? weight : 0, + cookie: cookieName(item.only_with_cookie), + weight: effectiveWeight(item.weight), disabled: toBoolean(item.disabled) || unchecked.length > 0, fnUuid: typeof config.fn_uuid === 'string' ? config.fn_uuid.toLowerCase() : null, managed: item.managed === true, @@ -154,9 +191,8 @@ function conditionWarning(rule) { /** * Pull the rules and the global headers of the project. - * @returns {Promise<{ rules: Object[], headers: Record|null, count: number, managed: number, warnings: string[] }>} - * rules: runtime rules in edge order (with the managed rules); count: the pulled rules; - * managed: the pulled rules that come from quant/edge-rules.json. + * @returns {Promise<{ rules: Object[], headers: Record|null, count: number, warnings: string[] }>} + * rules: runtime rules in edge order (with the managed rules); count: the pulled rules. */ export async function pullProject(client, { customer, project }) { const pulled = await pullRules(client, { customer, project }); @@ -168,7 +204,6 @@ export async function pullProject(client, { customer, project }) { rules, headers, count: rules.length, - managed: rules.filter((rule) => rule.managed).length, warnings: [...warnings, ...skipped.map(conditionWarning)] }; } diff --git a/src/serve/project.js b/src/serve/project.js index bbfd53e..ffa33b6 100644 --- a/src/serve/project.js +++ b/src/serve/project.js @@ -99,6 +99,7 @@ function ruleWarning(rule, fn, source) { * headers. The globalHeaders of quant/serve.json override them. * @returns {{ functions: Map, rules: Object[], stores: Object[], fixtures: string, inlineFunctions: boolean, warnings: string[] }} * functions is keyed by lowercase uuid; rules are runtime rules in run order. + * localRules: the number of rules from the rules manifest. * inlineFunctions: project_settings.enable_inline_fn in meta.json. * settings: the local project settings from serve.json (see project-config.js). */ @@ -115,6 +116,7 @@ export function loadProject({ root, functions = DEFAULT_FILES.functions, rules = return { functions: map, rules: runtimeRules, + localRules: localRules.length, stores: readStores(root, meta), fixtures: path.resolve(root, fixtures), inlineFunctions: readInlineSetting(root, meta), diff --git a/src/serve/rule-actions.js b/src/serve/rule-actions.js index 5538c6d..b7c4a06 100644 --- a/src/serve/rule-actions.js +++ b/src/serve/rule-actions.js @@ -232,14 +232,10 @@ export async function runAction(state, ctx, rule) { return action ? action(state, ctx, rule) : null; } -// A setting that is on: true, a number, or text or a list that is not empty. -const isOn = (value) => value === true || typeof value === 'number' || (typeof value === 'string' && value !== '') || - (Array.isArray(value) && value.length > 0); - -// TLS verification and failover work differently on this machine. Notification, +// TLS verification and failover work differently on this machine. The other +// failover settings have an effect only with failover_mode on. Notification, // alert and application settings have no local effect and get no warning. -const changesOriginTrust = (config) => config.disable_ssl_verify === true || - Object.keys(config).some((name) => name.startsWith('failover_') && isOn(config[name])); +const changesOriginTrust = (config) => config.disable_ssl_verify === true || config.failover_mode === true; function proxyWarnings(rule) { const config = configOf(rule); diff --git a/src/serve/rules.js b/src/serve/rules.js index ea4535f..89d752f 100644 --- a/src/serve/rules.js +++ b/src/serve/rules.js @@ -54,9 +54,15 @@ function methodAllowed(rule, method) { return !(rule.methodsNot && rule.methodsNot.includes(method)); } +// True when the Cookie header has a cookie with exactly this name. +function hasCookie(header, name) { + return String(header || '').split(';').some((pair) => pair.split('=')[0].trim() === name); +} + function ruleMatches(rule, ctx) { if (rule.disabled) return false; if (!methodAllowed(rule, ctx.method)) return false; + if (rule.cookie && !hasCookie(ctx.cookie, rule.cookie)) return false; if (!hostMatches(rule.domain, ctx.host)) return false; return rule.url.some(pattern => urlMatches(pattern, ctx.pathname, ctx.search || '')); } diff --git a/src/serve/server.js b/src/serve/server.js index f273062..485c252 100644 --- a/src/serve/server.js +++ b/src/serve/server.js @@ -183,7 +183,7 @@ function requestContext(state, req) { } function matchContext(ctx) { - return { host: ctx.host || '', method: ctx.method, pathname: ctx.pathname, search: ctx.search }; + return { host: ctx.host || '', method: ctx.method, pathname: ctx.pathname, search: ctx.search, cookie: ctx.req?.headers?.cookie || '' }; } function clientAddress(state, ctx) { @@ -714,7 +714,8 @@ function closer(state, server, cleanups) { * and pulledHeaders (--pull-rules: the portal global headers, or null). * @returns {Promise<{ url: string, close: () => Promise, setRunPid: (pid: number) => void, summary: Object }>} * setRunPid: the --run process group that the watchdog ends when a function blocks the server. - * summary: { stores, rules, libraries, warnings } for the start-up output. + * summary: { stores, rules, localRules, libraries, warnings } for the start-up output. + * localRules: the number of rules from the rules manifest. */ export async function startServer(options) { const state = createState(options); @@ -735,6 +736,7 @@ export async function startServer(options) { summary: { stores: storeList(state), rules: state.project.rules.map((rule) => ({ id: rule.id, type: rule.type || rule.action, urls: rule.url })), + localRules: state.project.localRules, libraries: fromProject, warnings: [site.warning, ...libraryErrors, ...state.project.warnings].filter(Boolean) } diff --git a/tests/unit/serve/pull-rules.test.mjs b/tests/unit/serve/pull-rules.test.mjs index df628fa..d3913fe 100644 --- a/tests/unit/serve/pull-rules.test.mjs +++ b/tests/unit/serve/pull-rules.test.mjs @@ -125,6 +125,34 @@ describe('serve --pull-rules', () => { expect(fromPortalRule(item(0, 'redirect', {}, { domain: [] })).domain).to.deep.equal(['any']); }); + it('takes the effective weight of the portal order', () => { + const weightOf = (weight) => fromPortalRule(item(0, 'redirect', {}, { weight })).weight; + expect(weightOf(25)).to.equal(25); + expect(weightOf(-7)).to.equal(-7); + expect(weightOf('42')).to.equal(42); + expect(weightOf('-3')).to.equal(-3); + expect(weightOf('1.0')).to.equal(1); + expect(weightOf('1e3')).to.equal(1000); + for (const value of [1.5, '1.5', true, false, 'abc', '', null, undefined, [], {}, Infinity]) { + expect(weightOf(value), String(value)).to.equal(0); + } + }); + + it('reads the cookie condition', () => { + expect(fromPortalRule(item(0, 'redirect', {}, { only_with_cookie: 'session' })).cookie).to.equal('session'); + for (const value of ['', null, undefined, 5]) { + expect(fromPortalRule(item(0, 'redirect', {}, { only_with_cookie: value })).cookie).to.equal(null); + } + }); + + it('takes an empty method list as all methods', () => { + const only = fromPortalRule(item(0, 'redirect', {}, { method: 'method_is', method_is: [] })); + expect(only.methods).to.equal(null); + const not = fromPortalRule(item(0, 'redirect', {}, { method: 'method_is_not', method_is_not: [] })); + expect(not.methodsNot).to.equal(null); + expect(fromPortalRule(item(0, 'redirect', {}, { method: 'ANY', method_is: ['GET'] })).methods).to.equal(null); + }); + it('reads the method condition', () => { const only = fromPortalRule(item(0, 'redirect', {}, { method: 'method_is', method_is: ['get', 'POST'] })); expect(only.methods).to.deep.equal(['GET', 'POST']); @@ -140,6 +168,16 @@ describe('serve --pull-rules', () => { expect(fromPortalRule(item(0, 'redirect')).unchecked).to.deep.equal([]); }); + it('takes an empty, any or empty-list condition as no condition', () => { + for (const over of [{ country: '' }, { country: null }, { country: undefined }, { country: 'ANY' }, { country: 'Any' }, + { country: 'country_is', country_is: [] }, { ip: 'ip_is_not', ip_is_not: [] }, { asn: 'asn_is' }]) { + const rule = fromPortalRule(item(0, 'redirect', {}, over)); + expect(rule.unchecked, JSON.stringify(over)).to.deep.equal([]); + expect(rule.disabled).to.equal(false); + } + expect(fromPortalRule(item(0, 'redirect', {}, { ua: 'ua_is_not', ua_is_not: ['bot'] })).unchecked).to.deep.equal(['ua']); + }); + it('makes booleans of boolean settings sent as text', () => { const rule = fromPortalRule(item(0, 'proxy', { only_proxy_404: 'true', static_error_page: '', waf_enabled: '0', disable_ssl_verify: '1', failover_mode: 'false' @@ -223,7 +261,7 @@ describe('serve --pull-rules', () => { const { calls, client } = fakePortal({ '/rules': reply(200, list), '/custom-headers': reply(200, { 'x-a': '1' }) }); const pulled = await pullProject(client, IDS); expect(calls).to.have.length(2); - expect(pulled).to.include({ count: 3, managed: 1 }); + expect(pulled.count).to.equal(3); expect(pulled.headers).to.deep.equal({ 'x-a': '1' }); expect(pulled.rules.map((rule) => rule.id)).to.deep.equal(['rule-0', 'rule-2', 'rule-4']); expect(pulled.warnings.filter((line) => line === PARTIAL_WARNING)).to.have.length(1); @@ -269,7 +307,10 @@ describe('serve --pull-rules', () => { } function localRules(url) { - return { version: 1, namespace: 'test', rules: [{ id: 'local-m', name: 'Local M', type: 'function', urls: [url], weight: 150, function_uuid: FN }] }; + return { version: 1, namespace: 'test', rules: [ + { id: 'local-m', name: 'Local M', type: 'function', urls: [url], weight: 150, function_uuid: FN }, + { id: 'local-z', name: 'Local Z', type: 'function', urls: ['/z'], weight: 900, function_uuid: FN } + ] }; } function projectRoot() { @@ -288,7 +329,8 @@ describe('serve --pull-rules', () => { item(1, 'auth', { auth_user: 'ada', auth_pass: PASSWORD }, { uuid: 'b', name: 'B', url: ['/staff'], weight: 100 }), item(2, 'edge_fn', { fn_uuid: FN }, { uuid: 'old', name: 'Old', url: ['/m'], weight: 150, managed: true }), item(4, 'proxy', { to: 'http://127.0.0.1:1$*', inject_headers: { 'x-shared': SECRET }, disable_ssl_verify: 'true' }, { uuid: 'p', name: 'P', url: ['/api/*'], weight: 160 }), - item(5, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'pulled C' }, { uuid: 'c', name: 'C', url: ['/m', '/n'], weight: 200 }) + item(5, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'pulled C' }, { uuid: 'c', name: 'C', url: ['/m', '/n'], weight: 200 }), + item(6, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'with cookie' }, { uuid: 'k', name: 'K', url: ['/cookie'], weight: 300, only_with_cookie: 'preview' }) ]; function portalFetch(calls) { @@ -335,7 +377,7 @@ describe('serve --pull-rules', () => { await until(() => url); try { expect(calls).to.have.length(2); - expect(output).to.include('Pulled 5 portal rules (1 from quant/edge-rules.json).'); + expect(output).to.include('Pulled 6 portal rules (2 from quant/edge-rules.json).'); expect(output.filter((line) => line.includes(PARTIAL_WARNING))).to.have.length(1); expect(output.join('\n')).to.include('Rule P: failover and TLS verification settings work differently in quant serve.'); expect((await fetch(`${url}/a`, { redirect: 'manual' })).headers.get('location')).to.equal('/moved'); @@ -343,6 +385,9 @@ describe('serve --pull-rules', () => { expect(await (await fetch(`${url}/m`)).text()).to.equal('local M'); expect(await (await fetch(`${url}/n`)).text()).to.equal('pulled C'); expect((await fetch(`${url}/`)).headers.get('x-portal')).to.equal(HEADER_SECRET); + expect((await fetch(`${url}/cookie`)).status).to.equal(404); + expect(await (await fetch(`${url}/cookie`, { headers: { cookie: 'preview=1' } })).text()).to.equal('with cookie'); + expect((await fetch(`${url}/cookie`, { headers: { cookie: 'preview_mode=1' } })).status).to.equal(404); // --watch: a change to quant/edge-rules.json merges again, with no new pull. await new Promise((resolve) => setTimeout(resolve, 300)); diff --git a/tests/unit/serve/rule-actions.test.mjs b/tests/unit/serve/rule-actions.test.mjs index d0dfef7..2fe4db0 100644 --- a/tests/unit/serve/rule-actions.test.mjs +++ b/tests/unit/serve/rule-actions.test.mjs @@ -371,7 +371,7 @@ describe('serve rule actions', () => { it('warns once, with no value, about failover and TLS verification settings', () => { const text = 'Rule API: failover and TLS verification settings work differently in quant serve.'; const tls = rule('proxy', ['*'], { to: 'https://o.test', disable_ssl_verify: true }, { id: 'a', name: 'API' }); - const failover = rule('proxy', ['*'], { to: 'https://o.test', failover_mode: true, failover_origin_ttfb: '2000' }, { id: 'b', name: 'API' }); + const failover = rule('proxy', ['*'], { to: 'https://o.test', failover_mode: true }, { id: 'b', name: 'API' }); const quiet = rule('proxy', ['*'], { to: 'https://o.test', disable_ssl_verify: false, failover_mode: false, failover_lifetime: null, failover_origin_status_codes: [], notify_config: { slack_webhook: 'https://hooks.test/x' }, proxy_alert_enabled: true, application_proxy: true, application_name: 'app' @@ -379,6 +379,10 @@ describe('serve rule actions', () => { expect(actionWarnings([tls])).to.deep.equal([text]); expect(actionWarnings([failover])).to.deep.equal([text]); expect(actionWarnings([quiet])).to.deep.equal([]); + const portalDefault = rule('proxy', ['*'], { + to: 'https://o.test', failover_mode: false, failover_lifetime: '300', failover_origin_ttfb: '5000', disable_ssl_verify: false + }, { id: 'd', name: 'Default' }); + expect(actionWarnings([portalDefault])).to.deep.equal([]); }); it('gives one warning for each rule id', () => { diff --git a/tests/unit/serve/rules.test.mjs b/tests/unit/serve/rules.test.mjs index 969ebff..bbca378 100644 --- a/tests/unit/serve/rules.test.mjs +++ b/tests/unit/serve/rules.test.mjs @@ -67,6 +67,16 @@ describe('serve rules', () => { expect(matchRule([r], ctx({ method: 'POST' }))).to.equal(r); }); + it('needs the cookie of a cookie rule, by its exact name', () => { + const r = rule({ cookie: 'session' }); + expect(matchRule([r], ctx())).to.equal(null); + expect(matchRule([r], ctx({ cookie: 'a=1; session=x' }))).to.equal(r); + expect(matchRule([r], ctx({ cookie: ' session = x ;b=2' }))).to.equal(r); + expect(matchRule([r], ctx({ cookie: 'session_id=x; sessionx=1' }))).to.equal(null); + expect(matchRule([r], ctx({ cookie: 'Session=x' }))).to.equal(null); + expect(matchRule([rule({ cookie: null })], ctx())).to.not.equal(null); + }); + it('refuses a method in the methodsNot list', () => { const r = rule({ methodsNot: ['DELETE'] }); expect(matchRule([r], ctx({ method: 'DELETE' }))).to.equal(null); From c036f3210dfe31464d124a47f1f97499daee74c1 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:28:21 +1000 Subject: [PATCH 18/32] feat(serve): read one remote KV store through the portal API --- src/serve/portal/client.js | 18 ++- src/serve/portal/kv-remote.js | 91 ++++++++++++++ tests/unit/serve/kv-remote-read.test.mjs | 151 +++++++++++++++++++++++ tests/unit/serve/portal-client.test.mjs | 29 +++++ 4 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 src/serve/portal/kv-remote.js create mode 100644 tests/unit/serve/kv-remote-read.test.mjs diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js index b2df90a..828fd3d 100644 --- a/src/serve/portal/client.js +++ b/src/serve/portal/client.js @@ -85,11 +85,14 @@ function refusal(status, scopeInfo) { return new PortalError(`QUANT_API_TOKEN lacks the ${scope} scope for ${what}.`, 403); } -async function send({ base, token, fetchImpl }, path, query) { +async function send({ base, token, fetchImpl }, { method, path, query, body }) { + const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' }; + if (body !== undefined) headers['Content-Type'] = 'application/json'; try { return await fetchImpl(buildUrl(base, path, query), { - method: 'GET', - headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' }, + method, + headers, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), redirect: 'manual', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) }); @@ -111,10 +114,10 @@ async function settle(context, response, scopeInfo) { export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) }) { const context = { base, token, fetchImpl }; - async function get(path, query, scopeInfo) { + async function request(call, scopeInfo) { let waited = 0; for (;;) { - const response = await send(context, path, query); + const response = await send(context, call); if (response.status !== 429) return settle(context, response, scopeInfo); const delay = retryDelayMs(response); if (waited + delay > MAX_WAIT_MS) throw new PortalError('The portal rate limit (429) did not clear in 30 seconds. Try again later.', 429); @@ -122,5 +125,8 @@ export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms await sleep(delay); } } - return { get }; + return { + get: (path, query, scopeInfo) => request({ method: 'GET', path, query }, scopeInfo), + post: (path, body, scopeInfo) => request({ method: 'POST', path, body }, scopeInfo) + }; } diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js new file mode 100644 index 0000000..dd4f791 --- /dev/null +++ b/src/serve/portal/kv-remote.js @@ -0,0 +1,91 @@ +import { PortalError } from './client.js'; + +const PAGE_LIMIT = 1000; +const BATCH_SIZE = 100; +const MASKED = '[ENCRYPTED]'; +const READ_SCOPE = 'kv:read'; + +const storePath = ({ customer, project }, leaf = '') => + `/organizations/${encodeURIComponent(customer)}/projects/${encodeURIComponent(project)}/kv${leaf}`; + +const notLinked = name => new PortalError(`The KV store ${name} is not linked to this project.`, 404); +const scopeInfo = name => ({ scope: READ_SCOPE, what: `KV store ${name}` }); + +export async function findStore(client, { customer, project, name }) { + const { status, body } = await client.get(storePath({ customer, project }), undefined, scopeInfo(name)); + const stores = status === 404 || !Array.isArray(body) ? [] : body; + const found = stores.find(store => store && store.name === name); + if (!found) throw notLinked(name); + return { id: found.id, name: found.name }; +} + +// Counts every request. `full` is true once the request cap is reached. +function budget(max, used) { + const state = { requests: used, max }; + state.full = () => state.requests >= state.max; + return state; +} + +async function fetchOk(state, name, send) { + state.requests += 1; + const { status, body } = await send(scopeInfo(name)); + if (status === 404) throw notLinked(name); + return body; +} + +function splitItem(item) { + const { key, value, ...metadata } = item; + return { key, value, metadata }; +} + +async function readValues(client, state, ctx, id, keys) { + const entries = []; + for (let start = 0; start < keys.length; start += BATCH_SIZE) { + if (state.full()) return { entries, complete: false }; + const chunk = keys.slice(start, start + BATCH_SIZE); + const body = await fetchOk(state, ctx.name, info => client.post(storePath(ctx, `/${encodeURIComponent(id)}/batch-get`), { keys: chunk }, info)); + const items = Array.isArray(body?.items) ? body.items : []; + for (const item of items) if (item && typeof item.key === 'string') entries.push(splitItem(item)); + } + return { entries, complete: true }; +} + +function readIndexes(client, state, ctx, id) { + return fetchOk(state, ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/indexes`), undefined, info)) + .then(body => (Array.isArray(body?.indexes) ? body.indexes : [])); +} + +function readKeysPage(client, state, ctx, id, cursor) { + const query = { limit: PAGE_LIMIT, ...(cursor ? { cursor } : {}) }; + return fetchOk(state, ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/keys`), query, info)); +} + +const pageKeys = page => (Array.isArray(page?.data) ? page.data : []).map(row => row?.key).filter(key => typeof key === 'string'); + +// Reads one page of keys and their values. Returns the next cursor, or a stop flag. +async function readPage(client, state, ctx, id, { cursor, room }) { + const page = await readKeysPage(client, state, ctx, id, cursor); + const all = pageKeys(page); + const values = await readValues(client, state, ctx, id, all.slice(0, room)); + const more = page?.has_more === true && Boolean(page.next_cursor); + const cut = !values.complete || room < all.length || (more && room <= all.length); + return { entries: values.entries, next: more ? page.next_cursor : null, cut }; +} + +export async function readStore(client, ctx, id, { maxKeys = 10000, maxRequests = 200 } = {}) { + const state = budget(maxRequests, ctx.requestsUsed ?? 0); + const entries = []; + const indexes = state.full() ? [] : await readIndexes(client, state, ctx, id); + let cursor; + let truncated = false; + do { + truncated = state.full(); + if (truncated) break; + const page = await readPage(client, state, ctx, id, { cursor, room: maxKeys - entries.length }); + entries.push(...page.entries); + truncated = page.cut; + cursor = truncated ? null : page.next; + } while (cursor); + const masked = entries.filter(entry => entry.value === MASKED).length; + return { entries, indexes, masked, truncated, requests: state.requests }; +} diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs new file mode 100644 index 0000000..7e63f01 --- /dev/null +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -0,0 +1,151 @@ +import { expect } from 'chai'; +import { findStore, readStore } from '../../../src/serve/portal/kv-remote.js'; +import { createPortalClient } from '../../../src/serve/portal/client.js'; + +const BASE = new URL('http://127.0.0.1:8080/api/v2'); +const CTX = { customer: 'org', project: 'proj', name: 'sessions' }; +const PREFIX = '/api/v2/organizations/org/projects/proj/kv'; + +const reply = (status, body) => ({ + status, + headers: { get: () => null }, + json: async () => { + if (body === undefined) throw new Error('no body'); + return body; + } +}); + +// A fake portal: pages of keys, batch-get, indexes and the store list. +function portal({ total = 0, pageSize = 1000, stores, indexes = [], status = {}, encrypted = [] } = {}) { + const calls = []; + const keys = Array.from({ length: total }, (_, i) => `k${String(i).padStart(5, '0')}`); + const fetchImpl = async (url, init) => { + const u = new URL(String(url)); + const route = u.pathname.replace(PREFIX, '') || '/'; + calls.push({ route, method: init.method, query: Object.fromEntries(u.searchParams), body: init.body && JSON.parse(init.body) }); + if (status[route]) return reply(status[route], { message: 'x' }); + if (route === '/') return reply(200, stores ?? [{ id: 'abc', name: 'sessions' }]); + if (route === '/abc/indexes') return reply(200, { indexes }); + if (route === '/abc/keys') { + const from = Number(u.searchParams.get('cursor') ?? 0); + const to = Math.min(from + Math.min(Number(u.searchParams.get('limit')), pageSize), total); + return reply(200, { data: keys.slice(from, to).map(key => ({ key })), next_cursor: to < total ? String(to) : null, has_more: to < total, page_end: 'limit' }); + } + if (route === '/abc/batch-get') { + return reply(200, { items: init.body && JSON.parse(init.body).keys.map(key => encrypted.includes(key) + ? { key, value: '[ENCRYPTED]', encrypted: true, version: 'V1' } + : { key, value: { n: key }, encrypted: false, version: 'V1', updated_at: 't' }) }); + } + return reply(404); + }; + return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {} }) }; +} + +const rejection = async (promise) => { + try { await promise; } catch (error) { return error; } + throw new Error('expected a rejection'); +}; + +describe('remote KV read', () => { + describe('findStore', () => { + it('matches the name exactly and returns id and name', async () => { + const { client, calls } = portal({ stores: [{ id: 'z', name: 'Sessions' }, { id: 'abc', name: 'sessions' }] }); + expect(await findStore(client, CTX)).to.deep.equal({ id: 'abc', name: 'sessions' }); + expect(calls[0].route).to.equal('/'); + }); + + it('fails when the store is not linked', async () => { + const { client } = portal({ stores: [{ id: 'z', name: 'other' }] }); + const error = await rejection(findStore(client, CTX)); + expect(error.message).to.equal('The KV store sessions is not linked to this project.'); + }); + + it('fails when the list is not an array or answers 404', async () => { + const bad = await rejection(findStore(portal({ stores: {} }).client, CTX)); + expect(bad.message).to.equal('The KV store sessions is not linked to this project.'); + const missing = await rejection(findStore(portal({ status: { '/': 404 } }).client, CTX)); + expect(missing.message).to.equal('The KV store sessions is not linked to this project.'); + }); + + it('names kv:read on 403', async () => { + const error = await rejection(findStore(portal({ status: { '/': 403 } }).client, CTX)); + expect(error.message).to.equal('QUANT_API_TOKEN lacks the kv:read scope for KV store sessions.'); + }); + }); + + describe('readStore', () => { + it('follows cursors with limit 1000', async () => { + const { client, calls } = portal({ total: 2500 }); + const result = await readStore(client, CTX, 'abc'); + expect(result.entries).to.have.length(2500); + const pages = calls.filter(c => c.route === '/abc/keys'); + expect(pages.map(c => c.query.cursor)).to.deep.equal([undefined, '1000', '2000']); + expect(pages[0].query.limit).to.equal('1000'); + expect(result.truncated).to.equal(false); + }); + + it('batches values at 100 keys per call', async () => { + const { client, calls } = portal({ total: 250 }); + const result = await readStore(client, CTX, 'abc'); + const batches = calls.filter(c => c.route === '/abc/batch-get'); + expect(batches.map(c => c.body.keys.length)).to.deep.equal([100, 100, 50]); + expect(batches.every(c => c.method === 'POST')).to.equal(true); + expect(result.entries[0]).to.deep.equal({ key: 'k00000', value: { n: 'k00000' }, metadata: { encrypted: false, version: 'V1', updated_at: 't' } }); + expect(result.requests).to.equal(calls.length); + expect(result.requests).to.equal(1 + 3 + 1); + }); + + it('reads index definitions and counts them as a request', async () => { + const indexes = [{ name: 'byX', collection: 'c', fields: ['x'], state: 'ACTIVE' }]; + const { client } = portal({ total: 1, indexes }); + const result = await readStore(client, CTX, 'abc'); + expect(result.indexes).to.deep.equal(indexes); + }); + + it('counts masked values and keeps the string', async () => { + const { client } = portal({ total: 3, encrypted: ['k00001'] }); + const result = await readStore(client, CTX, 'abc'); + expect(result.masked).to.equal(1); + expect(result.entries[1].value).to.equal('[ENCRYPTED]'); + }); + + it('drops keys that vanished between list and read', async () => { + const fetchImpl = async (url) => { + const route = new URL(String(url)).pathname.replace(PREFIX, ''); + if (route === '/abc/keys') return reply(200, { data: [{ key: 'a' }, { key: 'b' }], next_cursor: null, has_more: false }); + if (route === '/abc/batch-get') return reply(200, { items: [null, { key: 'b', value: 1 }] }); + return reply(200, { indexes: [] }); + }; + const client = createPortalClient({ base: BASE, token: 'tok', fetchImpl }); + const result = await readStore(client, CTX, 'abc'); + expect(result.entries).to.deep.equal([{ key: 'b', value: 1, metadata: {} }]); + }); + + it('stops at maxKeys and sets truncated', async () => { + const { client } = portal({ total: 2500 }); + const result = await readStore(client, CTX, 'abc', { maxKeys: 1500 }); + expect(result.entries).to.have.length(1500); + expect(result.truncated).to.equal(true); + }); + + it('stops at maxRequests and sets truncated', async () => { + const { client, calls } = portal({ total: 5000 }); + const result = await readStore(client, CTX, 'abc', { maxRequests: 4 }); + expect(calls).to.have.length(4); + expect(result.requests).to.equal(4); + expect(result.truncated).to.equal(true); + }); + + it('reports not linked on 404 from a store route', async () => { + const { client } = portal({ status: { '/abc/keys': 404 } }); + const error = await rejection(readStore(client, CTX, 'abc')); + expect(error.message).to.equal('The KV store sessions is not linked to this project.'); + }); + + it('names kv:read on 403 and never prints a value in errors', async () => { + const { client } = portal({ total: 1, status: { '/abc/batch-get': 403 } }); + const error = await rejection(readStore(client, CTX, 'abc')); + expect(error.message).to.equal('QUANT_API_TOKEN lacks the kv:read scope for KV store sessions.'); + }); + }); +}); diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index 0685b69..420231b 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -222,4 +222,33 @@ describe('portal client', () => { } }); }); + + describe('post', () => { + it('sends a JSON body with the bearer token', async () => { + const { calls, client } = setup([reply(200, { items: [] })]); + const result = await client.post('/x/batch-get', { keys: ['a'] }, { scope: 'kv:read', what: 'KV' }); + expect(result).to.deep.equal({ status: 200, body: { items: [] } }); + expect(calls[0].init.method).to.equal('POST'); + expect(calls[0].init.body).to.equal('{"keys":["a"]}'); + expect(calls[0].init.headers['Content-Type']).to.equal('application/json'); + expect(calls[0].init.headers.Authorization).to.equal(`Bearer ${TOKEN}`); + expect(calls[0].init.redirect).to.equal('manual'); + }); + + it('retries a 429, names the scope on 403 and returns 404', async () => { + const retry = setup([reply(429, {}, { 'retry-after': '1' }), reply(200, {})]); + await retry.client.post('/x', {}); + expect(retry.waits).to.deep.equal([1000]); + const denied = await failure(setup([reply(403, {})]).client.post('/x', {}, { scope: 'kv:read', what: 'KV store a' })); + expect(denied.message).to.equal('QUANT_API_TOKEN lacks the kv:read scope for KV store a.'); + expect(await setup([reply(404, {})]).client.post('/x', {})).to.deep.equal({ status: 404 }); + }); + + it('redacts the token and hides network errors', async () => { + const leaked = await failure(setup([reply(500, { message: `bad ${TOKEN}` })]).client.post('/x', {})); + expect(leaked.message).to.not.include(TOKEN); + const down = await failure(setup([new Error(`boom ${TOKEN}`)]).client.post('/x', {})); + expect(down.message).to.not.include(TOKEN); + }); + }); }); From fcd5dd36f5885561628fc1bc3253e03c26e6b9e5 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:35:15 +1000 Subject: [PATCH 19/32] fix(serve): count portal retries in the remote KV read cap The portal client calls an optional onRequest hook for each HTTP attempt, so readStore counts 429 retries in requests and in maxRequests. A keys page with has_more and no next_cursor now stops the read as truncated. --- src/serve/portal/client.js | 4 +- src/serve/portal/kv-remote.js | 22 +++++--- tests/unit/serve/kv-remote-read.test.mjs | 72 ++++++++++++++++++++++++ tests/unit/serve/portal-client.test.mjs | 30 ++++++++++ 4 files changed, 120 insertions(+), 8 deletions(-) diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js index 828fd3d..652495d 100644 --- a/src/serve/portal/client.js +++ b/src/serve/portal/client.js @@ -112,11 +112,13 @@ async function settle(context, response, scopeInfo) { return { status, body: body ?? null }; } -export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) }) { +// onRequest() is called once for each HTTP attempt, a 429 retry included. +export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), onRequest = () => {} }) { const context = { base, token, fetchImpl }; async function request(call, scopeInfo) { let waited = 0; for (;;) { + onRequest(); const response = await send(context, call); if (response.status !== 429) return settle(context, response, scopeInfo); const delay = retryDelayMs(response); diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js index dd4f791..ffcab67 100644 --- a/src/serve/portal/kv-remote.js +++ b/src/serve/portal/kv-remote.js @@ -19,15 +19,20 @@ export async function findStore(client, { customer, project, name }) { return { id: found.id, name: found.name }; } -// Counts every request. `full` is true once the request cap is reached. -function budget(max, used) { - const state = { requests: used, max }; +// Counts every request. With a counter (the onRequest hook of the client: +// { count }), the count is the counter, so 429 retries count too; else each +// call counts one, from `used`. `full` is true once the request cap is reached. +function budget(max, used, counter) { + const own = { count: used }; + const source = counter || own; + const state = { max, tick: () => { own.count += 1; } }; + Object.defineProperty(state, 'requests', { get: () => source.count }); state.full = () => state.requests >= state.max; return state; } async function fetchOk(state, name, send) { - state.requests += 1; + state.tick(); const { status, body } = await send(scopeInfo(name)); if (status === 404) throw notLinked(name); return body; @@ -67,13 +72,16 @@ async function readPage(client, state, ctx, id, { cursor, room }) { const page = await readKeysPage(client, state, ctx, id, cursor); const all = pageKeys(page); const values = await readValues(client, state, ctx, id, all.slice(0, room)); - const more = page?.has_more === true && Boolean(page.next_cursor); - const cut = !values.complete || room < all.length || (more && room <= all.length); + const hasMore = page?.has_more === true; + const more = hasMore && Boolean(page.next_cursor); + // has_more with no cursor: the rest cannot be read. + const lost = hasMore && !more; + const cut = !values.complete || room < all.length || (more && room <= all.length) || lost; return { entries: values.entries, next: more ? page.next_cursor : null, cut }; } export async function readStore(client, ctx, id, { maxKeys = 10000, maxRequests = 200 } = {}) { - const state = budget(maxRequests, ctx.requestsUsed ?? 0); + const state = budget(maxRequests, ctx.requestsUsed ?? 0, ctx.counter); const entries = []; const indexes = state.full() ? [] : await readIndexes(client, state, ctx, id); let cursor; diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs index 7e63f01..2bf48e4 100644 --- a/tests/unit/serve/kv-remote-read.test.mjs +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -142,6 +142,78 @@ describe('remote KV read', () => { expect(error.message).to.equal('The KV store sessions is not linked to this project.'); }); + it('reads one batch at exactly 100 keys and two at 101', async () => { + for (const [total, sizes] of [[100, [100]], [101, [100, 1]]]) { + const { client, calls } = portal({ total }); + const result = await readStore(client, CTX, 'abc'); + expect(calls.filter(c => c.route === '/abc/batch-get').map(c => c.body.keys.length)).to.deep.equal(sizes); + expect(result.entries).to.have.length(total); + expect(result.truncated).to.equal(false); + } + }); + + it('sets truncated when maxKeys ends on a page boundary and more pages exist', async () => { + const more = portal({ total: 2000 }); + const cut = await readStore(more.client, CTX, 'abc', { maxKeys: 1000 }); + expect(cut.entries).to.have.length(1000); + expect(cut.truncated).to.equal(true); + expect(more.calls.filter(c => c.route === '/abc/keys')).to.have.length(1); + const exact = await readStore(portal({ total: 1000 }).client, CTX, 'abc', { maxKeys: 1000 }); + expect(exact.entries).to.have.length(1000); + expect(exact.truncated).to.equal(false); + }); + + // A fake portal with one fixed keys page: `page` is the keys reply. + function onePage(page) { + const calls = []; + const fetchImpl = async (url, init) => { + const route = new URL(String(url)).pathname.replace(PREFIX, ''); + calls.push(route); + if (route === '/abc/keys') return reply(200, page); + if (route === '/abc/batch-get') return reply(200, { items: JSON.parse(init.body).keys.map(key => ({ key, value: 1 })) }); + return reply(200, { indexes: [] }); + }; + return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl }) }; + } + + it('stops with truncated when has_more is true and next_cursor is missing or empty', async () => { + for (const next of [undefined, null, '']) { + const { client, calls } = onePage({ data: [{ key: 'a' }], has_more: true, ...(next === undefined ? {} : { next_cursor: next }) }); + const result = await readStore(client, CTX, 'abc'); + expect(result.entries).to.have.length(1); + expect(result.truncated).to.equal(true); + expect(calls.filter(route => route === '/abc/keys')).to.have.length(1); + } + }); + + it('stops a repeated cursor at the request cap', async () => { + const { client, calls } = onePage({ data: [{ key: 'a' }], has_more: true, next_cursor: 'same' }); + const result = await readStore(client, CTX, 'abc', { maxRequests: 9 }); + expect(calls).to.have.length(9); + expect(result.requests).to.equal(9); + expect(result.truncated).to.equal(true); + }); + + it('counts 429 retries in requests and in the cap through onRequest', async () => { + const counter = { count: 0 }; + let limited = 0; + const fetchImpl = async (url, init) => { + const route = new URL(String(url)).pathname.replace(PREFIX, '') || '/'; + if (route === '/') return reply(200, [{ id: 'abc', name: 'sessions' }]); + if (route === '/abc/indexes' && limited++ < 2) return { status: 429, headers: { get: () => '0' }, json: async () => ({}) }; + if (route === '/abc/indexes') return reply(200, { indexes: [] }); + if (route === '/abc/keys') return reply(200, { data: [{ key: 'a' }], has_more: true, next_cursor: 'n' }); + return reply(200, { items: JSON.parse(init.body).keys.map(key => ({ key, value: 1 })) }); + }; + const client = createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {}, onRequest: () => { counter.count += 1; } }); + await findStore(client, CTX); + const result = await readStore(client, { ...CTX, counter }, 'abc', { maxRequests: 6 }); + // 1 store list + 3 index attempts + 1 keys page + 1 batch: the cap of 6 stops the read. + expect(counter.count).to.equal(6); + expect(result.requests).to.equal(6); + expect(result.truncated).to.equal(true); + }); + it('names kv:read on 403 and never prints a value in errors', async () => { const { client } = portal({ total: 1, status: { '/abc/batch-get': 403 } }); const error = await rejection(readStore(client, CTX, 'abc')); diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index 420231b..c53beeb 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -251,4 +251,34 @@ describe('portal client', () => { expect(down.message).to.not.include(TOKEN); }); }); + + describe('onRequest', () => { + function counted(responses) { + const queue = [...responses]; + let count = 0; + const fetchImpl = async () => { + const next = queue.shift(); + if (next instanceof Error) throw next; + return next; + }; + const client = createPortalClient({ base: BASE, token: TOKEN, fetchImpl, sleep: async () => {}, onRequest: () => { count += 1; } }); + return { client, count: () => count }; + } + + it('is called once for each HTTP attempt, retries included', async () => { + const get = counted([reply(429, {}, { 'retry-after': '1' }), reply(429, {}, { 'retry-after': '1' }), reply(200, {})]); + await get.client.get('/x'); + expect(get.count()).to.equal(3); + const post = counted([reply(429, {}, { 'retry-after': '1' }), reply(200, {})]); + await post.client.post('/x', {}); + expect(post.count()).to.equal(2); + }); + + it('counts a failed attempt and works without the hook', async () => { + const down = counted([new Error('boom')]); + await failure(down.client.get('/x')); + expect(down.count()).to.equal(1); + expect(await setup([reply(200, { ok: 1 })]).client.get('/x')).to.deep.equal({ status: 200, body: { ok: 1 } }); + }); + }); }); From 44fc315844576c0c1407623f719962b18e33443c Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:42:16 +1000 Subject: [PATCH 20/32] feat(serve): --kv-remote serves one portal KV store from a start-up snapshot --- README.md | 2 +- src/commands/serve.js | 80 ++++-- src/serve/kv/remote.js | 154 +++++++++++ src/serve/kv/wire.js | 57 +++- src/serve/portal/kv-remote.js | 2 +- tests/unit/commands/serve.test.mjs | 13 +- tests/unit/serve/kv-remote-read.test.mjs | 5 + tests/unit/serve/kv-remote.test.mjs | 318 +++++++++++++++++++++++ 8 files changed, 599 insertions(+), 32 deletions(-) create mode 100644 src/serve/kv/remote.js create mode 100644 tests/unit/serve/kv-remote.test.mjs diff --git a/README.md b/README.md index fa3e3ab..918c314 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ Flags: - `--clock` - `real` or `manual` (default: `real`). With `manual`, the time of the local KV stops at the start time and, in a session, moves only with `POST /_quant_serve/clock`. See [Local ledgers](#local-ledgers). - `--version-info` - Print the versions of the CLI and its KV and test packages, then exit. - `--pull-rules` - Read the rules of the portal project one time at start, and run them with the rules of `quant/edge-rules.json`. See [Portal rules](#portal-rules---pull-rules). -- `--kv-remote` - Not available yet. The command stops with a message. +- `--kv-remote ` - Read the KV store `` of the portal project one time at start (at most 10,000 keys) and serve it from memory. Writes stay local for the session, and a restart reads the portal again. The snapshot is never written to `.quant/`. Secret values read `[ENCRYPTED]`. The store must be in `quant/meta.json`, and the command needs `QUANT_API_TOKEN` and `QUANT_BASE_URL`. A key that `initial_values` reveals once is shown in a box when the store is first seeded. Save the key then. Quant serve also writes it to a file under `.quant/kv/`. diff --git a/src/commands/serve.js b/src/commands/serve.js index 9b170c2..f44e735 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -17,10 +17,13 @@ import { LEDGERS_FILE, readLedgerFile } from '../serve/kv/ledgers.js'; import { readServeConfig } from '../serve/project-config.js'; import { createPortalClient, portalOptions } from '../serve/portal/client.js'; import { pullProject } from '../serve/portal/pull-rules.js'; +import { readRemoteStore, remoteLines } from '../serve/kv/remote.js'; const DEFAULTS = Object.freeze({ dir: 'dist', port: 8787, host: '127.0.0.1', org: 'local', project: 'dev', userName: 'Owner' }); -const LATER_RELEASE = ['kv-remote']; -const PULL_IDENTITY = '--pull-rules needs the portal organisation and project: pass --org and --project, or set them in quant/serve.json or quant.json.'; +const LATER_RELEASE = []; +const IDENTITY_HELP = 'pass --org and --project, or set them in quant/serve.json or quant.json.'; +const PULL_IDENTITY = `--pull-rules needs the portal organisation and project: ${IDENTITY_HELP}`; +const REMOTE_IDENTITY = `--kv-remote needs the portal organisation and project: ${IDENTITY_HELP}`; const CLOCKS = ['real', 'manual']; const LOOPBACK_HOSTS = ['127.0.0.1', '::1', 'localhost']; const POSIX = process.platform !== 'win32'; @@ -139,17 +142,31 @@ function identityOptions(args, root, gitUserName, local) { }; } -// --pull-rules reads the rules of a real project, so the org and project -// must be named (the local defaults are not a portal project). -function pullOptions(args, root, local) { - if (args['pull-rules'] !== true) return { pullRules: false }; +// True when the org and project are named (the local defaults are not a portal project). +function portalIdentity(args, root, local) { const saved = readQuantJson(root); const org = first([args.org, args.clientid, local.org, saved.clientid], undefined); const project = first([args.project, local.project, saved.project], undefined); - if (org === undefined || project === undefined) throw new Error(PULL_IDENTITY); + return org !== undefined && project !== undefined; +} + +// --pull-rules reads the rules of a real project, so the org and project +// must be named. +function pullOptions(args, root, local) { + if (args['pull-rules'] !== true) return { pullRules: false }; + if (!portalIdentity(args, root, local)) throw new Error(PULL_IDENTITY); return { pullRules: true }; } +// --kv-remote reads one store of a real project. +function remoteOptions(args, root, local) { + const name = args['kv-remote']; + if (name === undefined || name === null || name === false) return { kvRemote: undefined }; + if (typeof name !== 'string' || name === '') throw new Error('--kv-remote needs the name of a KV store in quant/meta.json.'); + if (!portalIdentity(args, root, local)) throw new Error(REMOTE_IDENTITY); + return { kvRemote: name }; +} + function kvOptions(args) { return { reset: args.reset === true, @@ -187,6 +204,7 @@ export function resolveOptions(args, { root = process.cwd(), gitUserName = readG ...siteOptions(args, local), ...identityOptions(args, root, gitUserName, local), ...pullOptions(args, root, local), + ...remoteOptions(args, root, local), ...kvOptions(args), ...devOptions(args) }; @@ -236,12 +254,13 @@ function clockLines(clock) { * rules, the edge libraries from the project, the warnings, then the * revealed keys in a box. */ -export function startupLines({ url, host, summary, keys = [], ledgers = [], clock = 'real', pull = null }) { +export function startupLines({ url, host, summary, keys = [], ledgers = [], clock = 'real', pull = null, remote = [] }) { const libraries = summary.libraries.length > 0 ? summary.libraries.join(', ') : 'none'; return [ `quant serve is running at ${url}`, ...pullLines(pull), ...section('KV stores:', summary.stores.map((store) => `${store.name} → ${store.location || store.id}`)), + ...remote, ...ledgerLines(ledgers, clock), ...clockLines(clock), ...section('Rules:', summary.rules.map((rule) => `${rule.id} ${rule.type} ${rule.urls.join(' ')}`)), @@ -415,15 +434,32 @@ function backendClock(mode) { return mode === 'manual' ? { clock: createVirtualClock(Date.now()) } : {}; } +// --kv-remote: the store must be in meta.json (so functions may open it). +// The portal is read before the local state is opened. +async function remoteSnapshot(options, stores, portal) { + const name = options.kvRemote; + if (!name) return null; + if (!stores.some((store) => store.name === name)) throw new Error(`--kv-remote ${name} is not a KV store in quant/meta.json.`); + return readRemoteStore({ name, org: options.org, project: options.project }, portal); +} + +// The start-up lines of the remote store, after it is in the local KV. +async function loadRemote(kv, snapshot, ledgers) { + if (!snapshot) return null; + const built = await kv.loadRemote(snapshot); + return { name: snapshot.name, lines: remoteLines(snapshot, built, ledgers) }; +} + // The local KV of the project (unless the caller gives a kvStore): open the // state, declare the meta.json stores and indexes and the local ledgers, -// then seed new stores. -async function localKv(options, log, print) { +// then seed new stores, then put the --kv-remote snapshot in memory. +async function localKv(options, { log, print, portal }) { const files = { root: options.root, functions: options.functions, rules: options.rules, meta: options.meta, fixtures: options.fixtures }; const { stores, fixtures } = loadProject(files); const ledgers = readLedgerFile({ root: options.root, stores, meta: options.meta }); + const snapshot = await remoteSnapshot(options, stores, portal); const kv = await openLocalKv({ - root: options.root, org: options.org, project: options.project, stores, ledgers, + root: options.root, org: options.org, project: options.project, stores, ledgers, remote: options.kvRemote, memory: options.memory, reset: options.reset, lagMs: options.kvLag, indexState: options.indexes, warn: log, ledgerWait: options.clock !== 'manual', realTime: options.clock === 'real', ...backendClock(options.clock) }); @@ -437,9 +473,10 @@ async function localKv(options, log, print) { throw err; } const keys = withFile(seeded.revealed); + const remote = await loadRemote(kv, snapshot, ledgers); return { kvStore: kv.kvStore, backend: kv.backend, ledgers, keys, warnings: kv.warnings, - location: kv.paths ? kv.paths.file : 'memory' + location: kv.paths ? kv.paths.file : 'memory', remote }; } @@ -477,8 +514,14 @@ function stderrLine(line) { } // The kvStore that the caller gives, else the local KV. -async function kvFor(options, kvStore, log, print) { - return kvStore ? { kvStore, given: true } : localKv(options, log || stderrLine, print); +async function kvFor(options, { kvStore, log, print, portal }) { + return kvStore ? { kvStore, given: true } : localKv(options, { log: log || stderrLine, print, portal }); +} + +// The KV state location of a store in the start-up lines. +function storeLocation(store, kv) { + if (kv.remote && store.name === kv.remote.name) return `${store.id} in memory (snapshot from the portal)`; + return `${store.id} in ${kv.location}`; } // The start-up summary and keys, with the KV state location, warnings, ledgers and revealed keys. @@ -486,9 +529,10 @@ function startupSummary(summary, kv) { if (kv.given) return { summary, keys: [] }; return { ledgers: kv.ledgers, + remote: kv.remote ? kv.remote.lines : [], summary: { ...summary, - stores: summary.stores.map((store) => ({ ...store, location: `${store.id} in ${kv.location}` })), + stores: summary.stores.map((store) => ({ ...store, location: storeLocation(store, kv) })), warnings: [...summary.warnings, ...kv.warnings] }, keys: kv.keys @@ -524,12 +568,12 @@ function watchRun(server, child) { * (default: SIGINT, SIGTERM or SIGHUP). * @param {Object} options From resolveOptions(). * @param {Object} [deps] stop, print, log, kvStore (replaces the local KV), onListening, - * env (QUANT_API_TOKEN and QUANT_BASE_URL for --pull-rules) and fetchImpl (the portal fetch). + * env (QUANT_API_TOKEN and QUANT_BASE_URL for --pull-rules and --kv-remote) and fetchImpl (the portal fetch). */ export async function runServe(options, { stop, print = console.log, log, kvStore, onListening, env, fetchImpl } = {}) { if (options.build) runBuild(options.build, options.root); const pulled = await pulledProject(options, { env, fetchImpl }); - const kv = await kvFor(options, kvStore, log, print); + const kv = await kvFor(options, { kvStore, log, print, portal: { env, fetchImpl } }); const server = await startOrShowKeys(serverOptions(withPull(options, pulled), { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); const pullWarnings = pulled ? pulled.warnings : []; const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...pullWarnings, ...server.summary.warnings] }; @@ -619,7 +663,7 @@ export default { .option('run', { type: 'string', describe: 'Long-running command (a dev server) to start beside the server' }) .option('proxy', { type: 'string', describe: 'Dev server URL for requests that no rule ends' }) .option('pull-rules', { type: 'boolean', describe: 'Read the portal rules at start and run them with the local rules (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) - .option('kv-remote', { type: 'string', describe: 'Read-only remote KV store (a later release)' }) + .option('kv-remote', { type: 'string', describe: 'Serve one portal KV store from a snapshot read at start; writes stay local (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) .option('version-info', { type: 'boolean', describe: 'Print the versions and exit' }), promptArgs: async () => { const dir = await text({ message: 'Directory of the built site', placeholder: DEFAULTS.dir, defaultValue: DEFAULTS.dir }); diff --git a/src/serve/kv/remote.js b/src/serve/kv/remote.js new file mode 100644 index 0000000..d6082b9 --- /dev/null +++ b/src/serve/kv/remote.js @@ -0,0 +1,154 @@ +/** + * --kv-remote: one KV store of the portal, read one time at start into + * memory. Local writes go on top of the snapshot for the session. Nothing + * of the snapshot is written to .quant/ or to any other file: a restart + * reads the portal again. Secret values stay the string [ENCRYPTED]. + */ + +import { createLocalKv } from '@quantcdn/edge-kv'; +import { createPortalClient, portalOptions } from '../portal/client.js'; +import { findStore, readStore } from '../portal/kv-remote.js'; + +export const REMOTE_MAX_KEYS = 10000; +const MASKED = '[ENCRYPTED]'; +// Portal index states that do not serve queries: they are not declared. +const UNUSED_STATES = ['FAILED', 'DROPPING']; + +/** + * The local declare shape of a portal index definition: name, collection, + * fields, sortBy, unique and count. The state fields (slot, state, createdAt, + * error, conflicts, job) are dropped. + */ +export function toLocalIndex(def) { + const out = { + name: def.name, + collection: def.collection, + fields: Array.isArray(def.fields) ? [...def.fields] : [], + unique: def.unique === true, + count: def.count === true + }; + if (typeof def.sortBy === 'string' && def.sortBy !== '') out.sortBy = def.sortBy; + return out; +} + +/** + * Read the store from the portal: the store list, the index definitions, + * the keys and their values. The request count includes 429 retries. + * A PortalError stops the start-up with its one-line message. + */ +export async function readRemoteStore({ name, org, project }, { env = process.env, fetchImpl } = {}) { + const ids = portalOptions({ clientid: org, project }, env); + const counter = { count: 0 }; + const client = createPortalClient({ + base: ids.base, token: ids.token, onRequest: () => { counter.count += 1; }, ...(fetchImpl ? { fetchImpl } : {}) + }); + const ctx = { customer: ids.customer, project: ids.project, name }; + const store = await findStore(client, ctx); + const snapshot = await readStore(client, { ...ctx, counter }, store.id, { maxKeys: REMOTE_MAX_KEYS }); + return { name, format: store.format ?? 2, ...snapshot }; +} + +// The set options of a portal entry: secret for an encrypted value, and the +// time left of its expiry (epoch seconds). null when the entry has expired. +function putOptions(metadata, nowMs) { + const options = metadata.encrypted === true ? { secret: true } : {}; + if (typeof metadata.expires_at !== 'number') return options; + const ttl = metadata.expires_at - nowMs / 1000; + return ttl > 0 ? { ...options, ttl } : null; +} + +// A ledger entry, or an entry with no value, is not copied. +const copyable = (entry) => entry.value !== undefined && entry.metadata?.ledger === undefined; + +async function writeEntry(backend, id, entry, result) { + const options = putOptions(entry.metadata || {}, backend.now()); + if (!copyable(entry) || options === null) return; + try { + await backend.put(id, entry.key, JSON.stringify(entry.value), options); + result.copied += 1; + if (entry.value === MASKED) result.masked += 1; + } catch (err) { + result.failed.add(err.code || 'ERROR'); + } +} + +async function declareIndexes(backend, id, { name, indexes }, { building, warnings }) { + for (const def of indexes.filter((item) => !UNUSED_STATES.includes(String(item?.state).toUpperCase()))) { + const local = toLocalIndex(def); + try { + await backend.declareIndex(id, local); + if (building) await backend.setIndexState(id, local.name, 'BUILDING'); + } catch (err) { + warnings.push(`KV store ${name}: index ${local.name} from the portal could not be declared locally (${err.code || err.message}).`); + } + } +} + +/** + * The remote store as store data of a backend snapshot. It is built in a + * backend in memory that starts from `base` (the keys and clock offset of + * the local backend) with no stores: the index definitions first, then each + * entry with its metadata (secret, ttl). + * @returns {Promise<{ data: Object, copied: number, masked: number, warnings: string[] }>} + */ +export async function buildRemoteStore({ base, id, snapshot, backendOptions, building = false }) { + const { backend } = createLocalKv({ ...backendOptions, storage: undefined }); + backend.restore({ ...base, stores: [] }); + await backend.createStore({ id, format: snapshot.format }); + const warnings = []; + await declareIndexes(backend, id, snapshot, { building, warnings }); + const result = { copied: 0, masked: 0, failed: new Set() }; + for (const entry of snapshot.entries) await writeEntry(backend, id, entry, result); + if (result.failed.size > 0) { + warnings.push(`KV store ${snapshot.name}: some keys from the portal could not be copied locally (${[...result.failed].join(', ')}).`); + } + const data = backend.snapshot().stores.find((store) => store.id === id); + return { data, copied: result.copied, masked: result.masked, warnings }; +} + +/** The start-up lines of the remote store. */ +export function remoteLines({ name, truncated }, { copied, masked }, ledgers) { + const lines = [truncated + ? `KV store ${name}: copied the first ${copied} keys only (limit ${REMOTE_MAX_KEYS}). Writes stay local.` + : `KV store ${name}: copied ${copied} keys from the portal (${masked} encrypted values masked).`]; + const declared = ledgers.some((entry) => entry.store === name && entry.defs.length > 0); + if (declared) lines.push(`Ledgers of ${name} are not copied from the portal; declare local ledgers in quant/kv-ledgers.json.`); + return lines; +} + +const without = (doc, id) => doc.stores.filter((store) => store.id !== id); + +// The document with the store `id` replaced by data (or removed when data is null). +function splice(doc, id, data) { + if (!doc) return doc; + return { ...doc, stores: data ? [...without(doc, id), data] : without(doc, id) }; +} + +/** + * A storage hook over the file storage that keeps one store in memory. + * A load gives the disk state with the store of the session; a save writes + * the disk copy of that store back (or none), never the store of the session. + * @returns {{ storage: Object, setRemote: (data: Object) => void }} + */ +export function remoteOverlay(storage, id) { + let remote = null; + const withRemote = (doc) => (remote ? splice(doc, id, remote) : doc); + return { + storage: { + load: () => withRemote(storage.load()), + changed: () => storage.changed(), + transaction: (run) => storage.transaction(async (fresh) => { + const disk = fresh ? fresh.stores.find((store) => store.id === id) ?? null : null; + const next = await run(withRemote(fresh)); + if (remote) remote = next.stores.find((store) => store.id === id) ?? remote; + return splice(next, id, disk); + }) + }, + setRemote: (data) => { remote = data; } + }; +} + +/** The snapshot with the store data in place of any store with the same id. */ +export function withStore(snapshot, data) { + return splice(snapshot, data.id, data); +} diff --git a/src/serve/kv/wire.js b/src/serve/kv/wire.js index 8721bbd..003a291 100644 --- a/src/serve/kv/wire.js +++ b/src/serve/kv/wire.js @@ -9,6 +9,7 @@ import { createLocalKv } from '@quantcdn/edge-kv'; import { storeId } from '../../helper/quant-meta.js'; import { seedStore } from '../seed.js'; import { declareLedgers, retiredLedgerWarnings } from './ledgers.js'; +import { buildRemoteStore, remoteOverlay, withStore } from './remote.js'; import path from 'node:path'; import { statePaths, ensureQuantDir, fileStorage, readBackendKeys, replaceState, writeBackendKeys, resetState @@ -143,10 +144,12 @@ async function seedStores(ctx, { fixtures, userName, seedKey, now }) { } // The meta.json stores and indexes, then the quant/kv-ledgers.json ledgers. +// The remote store (--kv-remote) is left out: loadRemote declares it. async function declareAll(ctx, { stores = [], ledgers = [], ledgerWait = true }) { - for (const definition of stores) await declareStore(ctx, definition); + const local = (name) => !ctx.remote || name !== ctx.remote.name; + for (const definition of stores.filter((store) => local(store.name))) await declareStore(ctx, definition); await declareLedgers(ctx.backend, { - org: ctx.org, project: ctx.project, ledgers, wait: ledgerWait, warnings: ctx.warnings + org: ctx.org, project: ctx.project, ledgers: ledgers.filter((entry) => local(entry.store)), wait: ledgerWait, warnings: ctx.warnings }); ctx.warnings.push(...retiredLedgerWarnings(ctx.backend, [...ctx.allowed.keys()])); } @@ -182,6 +185,43 @@ function backendOptions(paths, storage, options) { }; } +// The remote store of the session (--kv-remote), or null. +function remoteOf({ remote, stores = [], ledgers = [], org, project }) { + if (!remote) return null; + return { + name: remote, + id: storeId(org, project, remote), + definition: stores.find((store) => store.name === remote), + ledgers: ledgers.filter((entry) => entry.store === remote) + }; +} + +// --kv-remote: put the snapshot store in the backend (in memory only), allow +// it, then declare its local ledgers. The seed has run already and skipped it. +async function loadRemote(ctx, snapshot) { + const { id, definition, ledgers } = ctx.remote; + const built = await buildRemoteStore({ + base: ctx.backend.snapshot(), id, snapshot, + backendOptions: backendOptions(null, undefined, ctx.options), building: ctx.indexState === 'BUILDING' + }); + if (ctx.overlay) ctx.overlay.setRemote(built.data); + ctx.backend.restore(withStore(ctx.backend.snapshot(), built.data)); + ctx.allowed.set(id, definition); + await declareLedgers(ctx.backend, { + org: ctx.org, project: ctx.project, ledgers, wait: ctx.options.ledgerWait !== false, warnings: ctx.warnings + }); + ctx.warnings.push(...built.warnings); + return built; +} + +// The file storage; with a remote store, the overlay that keeps it in memory. +function storageOf(paths, remote) { + if (!paths) return { storage: undefined, overlay: null }; + if (!remote) return { storage: fileStorage(paths), overlay: null }; + const overlay = remoteOverlay(fileStorage(paths), remote.id); + return { storage: overlay.storage, overlay }; +} + /** * Open the local KV of a project. * @param {Object} options @@ -198,18 +238,22 @@ function backendOptions(paths, storage, options) { * @param {boolean} [options.realTime] Clear a stored clock offset before the declarations (--clock real). * @param {Object} [options.clock] The clock of the backend (`{ now() }`). Default: the real clock. * @param {(message: string) => void} [options.warn] Unknown set and delete options. + * @param {string} [options.remote] The --kv-remote store name: it is not declared or seeded here, + * and loadRemote(snapshot) puts it in memory only. * @returns {Promise<{ kvStore, backend, warnings: string[], paths: Object|null, offsetCleared: boolean, - * declare: (store: Object) => Promise, seed: (options: Object) => Promise }>} + * declare: (store: Object) => Promise, seed: (options: Object) => Promise, + * loadRemote: (snapshot: Object) => Promise }>} */ export async function openLocalKv(options) { const { root, org, project, memory = false, reset = false, warn } = options; const paths = memory ? null : await prepareFiles(root, reset); const allowed = new Map(); - const storage = paths ? fileStorage(paths) : undefined; + const remote = remoteOf(options); + const { storage, overlay } = storageOf(paths, remote); const { kvStore, backend } = createLocalKv({ ...backendOptions(paths, storage, options), isAllowed: (id) => allowed.has(id), warn }); if (paths && readBackendKeys(paths) === undefined) writeBackendKeys(paths, backend.snapshot().keys); const ctx = { - org, project, backend, kvStore, paths, allowed, options, warn, + org, project, backend, kvStore, paths, allowed, options, warn, remote, overlay, indexState: options.indexState === undefined ? null : indexState(options.indexState), warnings: [], seededInMemory: {} }; const offsetCleared = await clearClockOffset(backend, options.realTime === true); @@ -221,6 +265,7 @@ export async function openLocalKv(options) { offsetCleared, warnings: ctx.warnings, declare: (definition) => declareStore(ctx, definition), - seed: (seedOptions = {}) => seedAll(ctx, seedOptions) + seed: (seedOptions = {}) => seedAll(ctx, seedOptions), + loadRemote: (snapshot) => loadRemote(ctx, snapshot) }; } diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js index ffcab67..5067221 100644 --- a/src/serve/portal/kv-remote.js +++ b/src/serve/portal/kv-remote.js @@ -16,7 +16,7 @@ export async function findStore(client, { customer, project, name }) { const stores = status === 404 || !Array.isArray(body) ? [] : body; const found = stores.find(store => store && store.name === name); if (!found) throw notLinked(name); - return { id: found.id, name: found.name }; + return { id: found.id, name: found.name, ...([1, 2].includes(found.format) ? { format: found.format } : {}) }; } // Counts every request. With a counter (the onRequest hook of the client: diff --git a/tests/unit/commands/serve.test.mjs b/tests/unit/commands/serve.test.mjs index 24ed041..d8f9e28 100644 --- a/tests/unit/commands/serve.test.mjs +++ b/tests/unit/commands/serve.test.mjs @@ -197,12 +197,13 @@ describe('serve command', () => { expect(() => resolveOptions({ clock: 'fast' }, { root: tmp, gitUserName: noGit })).to.throw('--clock must be real or manual'); }); - for (const flag of [['--kv-remote', 'orbit']]) { - it(`stops for ${flag[0]}, which is available in a later release`, () => { - expect(() => resolveOptions(parse(flag), { root: tmp, gitUserName: noGit })) - .to.throw(`${flag[0]} is available in a later release`); - }); - } + it('takes --kv-remote with the portal org and project, and refuses it without them', () => { + const flags = ['--kv-remote', 'orbit', '--org', 'acme', '--project', 'site']; + expect(resolveOptions(parse(flags), { root: tmp, gitUserName: noGit }).kvRemote).to.equal('orbit'); + expect(resolveOptions(parse([]), { root: tmp, gitUserName: noGit }).kvRemote).to.equal(undefined); + expect(() => resolveOptions(parse(['--kv-remote', 'orbit']), { root: tmp, gitUserName: noGit })) + .to.throw('--kv-remote needs the portal organisation and project'); + }); it('refuses --kv-lag that is not a whole number of milliseconds', () => { expect(() => resolveOptions(parse(['--kv-lag', '1.5']), { root: tmp, gitUserName: noGit })).to.throw('--kv-lag'); diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs index 2bf48e4..91f0bab 100644 --- a/tests/unit/serve/kv-remote-read.test.mjs +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -54,6 +54,11 @@ describe('remote KV read', () => { expect(calls[0].route).to.equal('/'); }); + it('passes the store format through when the list reports 1 or 2', async () => { + expect(await findStore(portal({ stores: [{ id: 'abc', name: 'sessions', format: 1 }] }).client, CTX)).to.deep.equal({ id: 'abc', name: 'sessions', format: 1 }); + expect(await findStore(portal({ stores: [{ id: 'abc', name: 'sessions', format: 'x' }] }).client, CTX)).to.deep.equal({ id: 'abc', name: 'sessions' }); + }); + it('fails when the store is not linked', async () => { const { client } = portal({ stores: [{ id: 'z', name: 'other' }] }); const error = await rejection(findStore(client, CTX)); diff --git a/tests/unit/serve/kv-remote.test.mjs b/tests/unit/serve/kv-remote.test.mjs new file mode 100644 index 0000000..2c243bb --- /dev/null +++ b/tests/unit/serve/kv-remote.test.mjs @@ -0,0 +1,318 @@ +import { expect } from 'chai'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { resolveOptions, runServe } from '../../../src/commands/serve.js'; +import { toLocalIndex } from '../../../src/serve/kv/remote.js'; + +const TOKEN = 'remote-token-never-printed'; +const ENV = { QUANT_API_TOKEN: TOKEN, QUANT_BASE_URL: 'http://127.0.0.1:8080/api/v2' }; +const PREFIX = '/api/v2/organizations/acme/projects/site/kv'; +const FN = 'aaaa0000-0000-4000-8000-0000000000bb'; +const SNAPSHOT_VALUE = 'snapshot-value-never-on-disk'; +const LOCAL_WRITE = 'local-write-never-on-disk'; +const SNAPSHOT_STORE = 'acme-site-sessions'; + +const reply = (status, body) => ({ status, headers: { get: () => null }, json: async () => body }); + +// The remote store: three index members, a masked secret and a key with a ttl. +function remoteItems() { + return [ + { key: 'item:1', value: { color: 'red', note: SNAPSHOT_VALUE }, encrypted: false, version: 'V1' }, + { key: 'item:2', value: { color: 'red' }, encrypted: false, version: 'V1' }, + { key: 'item:3', value: { color: 'blue' }, encrypted: false, version: 'V1' }, + { key: 'api-key', value: '[ENCRYPTED]', encrypted: true, version: 'V1' }, + { key: 'short', value: 'lives', encrypted: false, version: 'V1', expires_at: Math.floor(Date.now() / 1000) + 3600 } + ]; +} + +const INDEXES = [ + { name: 'byColor', collection: 'item', fields: ['color'], unique: false, count: true, slot: 1, state: 'ACTIVE', createdAt: '2026-11-01T00:00:00.000Z' } +]; + +// A fake portal for one store "sessions" (id abc). `status` maps a route to a status. +function fakePortal({ items = remoteItems(), status = {}, stores } = {}) { + const calls = []; + const byKey = new Map(items.map((item) => [item.key, item])); + const keys = items.map((item) => item.key); + const fetchImpl = async (url, init) => { + const u = new URL(String(url)); + const route = u.pathname.replace(PREFIX, '') || '/'; + calls.push({ route, auth: init.headers.Authorization }); + if (status[route]) return reply(status[route], { message: 'refused' }); + if (route === '/') return reply(200, stores ?? [{ id: 'abc', name: 'sessions' }, { id: 'zzz', name: 'local' }]); + if (route === '/abc/indexes') return reply(200, { indexes: INDEXES }); + if (route === '/abc/keys') { + const from = Number(u.searchParams.get('cursor') ?? 0); + const to = Math.min(from + Number(u.searchParams.get('limit')), keys.length); + return reply(200, { data: keys.slice(from, to).map((key) => ({ key })), next_cursor: to < keys.length ? String(to) : null, has_more: to < keys.length }); + } + if (route === '/abc/batch-get') return reply(200, { items: JSON.parse(init.body).keys.map((key) => byKey.get(key) ?? null) }); + return reply(404, {}); + }; + return { calls, fetchImpl }; +} + +// The function: ?op=get|set|count|list&store=&key=&value=&color= +const FUNCTION = ` +const url = new URL(event.request.url); +const q = (name) => url.searchParams.get(name); +const store = kvStore.open('acme-site-' + (q('store') || 'sessions')); +let out; +if (q('op') === 'set') out = await store.set(q('key'), q('value')); +else if (q('op') === 'count') out = await store.count({ index: 'byColor', equals: { color: q('color') } }); +else if (q('op') === 'list') out = (await store.list({ index: 'byColor', equals: { color: q('color') } })).items.map((item) => item.key); +else out = await store.get(q('key')); +return new Response(JSON.stringify(out === undefined ? null : out), { headers: { 'x-version': 'one' } }); +`; + +function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(value, null, 2)); +} + +function projectRoot(tmp, { ledgers } = {}) { + const root = path.join(tmp, 'site'); + fs.mkdirSync(path.join(root, 'dist'), { recursive: true }); + fs.writeFileSync(path.join(root, 'dist', 'index.html'), 'home'); + fs.mkdirSync(path.join(root, 'quant', 'edge-functions'), { recursive: true }); + fs.writeFileSync(path.join(root, 'quant', 'edge-functions', 'kv.js'), FUNCTION); + writeJson(path.join(root, 'quant', 'edge-functions.json'), [{ id: 'kv', uuid: FN, type: 'function', path: 'quant/edge-functions/kv.js', description: 'kv' }]); + writeJson(path.join(root, 'quant', 'edge-rules.json'), { version: 1, namespace: 'test', rules: [ + { id: 'kv', name: 'KV', type: 'function', urls: ['/kv'], weight: 100, function_uuid: FN } + ] }); + writeJson(path.join(root, 'quant', 'meta.json'), { kv_stores: [ + { name: 'sessions', initial_values: [{ key: 'seeded', value: 'from meta' }] }, + { name: 'local', initial_values: [{ key: 'own', value: 'mine' }] } + ] }); + fs.mkdirSync(path.join(root, 'quant', 'kv-fixtures'), { recursive: true }); + writeJson(path.join(root, 'quant', 'kv-fixtures', 'sessions.json'), [{ key: 'fixture', value: 'from fixtures' }]); + if (ledgers) writeJson(path.join(root, 'quant', 'kv-ledgers.json'), ledgers); + return root; +} + +function filesUnder(dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + return entry.isDirectory() ? filesUnder(full) : [full]; + }); +} + +async function until(check, timeoutMs = 8000) { + const started = Date.now(); + while (!(await check())) { + if (Date.now() - started > timeoutMs) throw new Error('timed out'); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +// Start quant serve; returns { url, output, ask(query), close() }. +async function serve(root, portal, flags = {}) { + const output = []; + let stop; + const stopped = new Promise((resolve) => { stop = resolve; }); + let url; + const options = resolveOptions({ dir: 'dist', port: 0, org: 'acme', project: 'site', 'kv-remote': 'sessions', ...flags }, { root, gitUserName: () => null }); + const running = runServe(options, { + stop: stopped, + print: (line) => output.push(line), + log: (line) => output.push(line), + env: ENV, + fetchImpl: portal.fetchImpl, + onListening: (server) => { url = server.url; } + }); + let failed = null; + running.catch((err) => { failed = err; }); + await until(() => url || failed); + if (failed) throw failed; + const ask = async (query) => JSON.parse(await (await fetch(`${url}/kv?${new URLSearchParams(query)}`)).text()); + return { url, output, ask, close: async () => { stop(); await running; } }; +} + +async function failure(promise) { + try { + await promise; + } catch (error) { + return error; + } + throw new Error('expected a rejection'); +} + +describe('serve --kv-remote', function () { + this.timeout(20000); + let tmp; + + beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-kv-remote-')); }); + afterEach(() => fs.rmSync(tmp, { recursive: true, force: true })); + + describe('toLocalIndex', () => { + it('keeps the definition fields of a portal index and drops the state fields', () => { + expect(toLocalIndex(INDEXES[0])).to.deep.equal({ name: 'byColor', collection: 'item', fields: ['color'], unique: false, count: true }); + expect(toLocalIndex({ name: 'byDate', collection: 'post', sortBy: 'date', state: 'ACTIVE' })) + .to.deep.equal({ name: 'byDate', collection: 'post', fields: [], sortBy: 'date', unique: false, count: false }); + }); + }); + + it('serves the snapshot, keeps writes local, and runs no seed for the remote store', async () => { + const root = projectRoot(tmp); + const portal = fakePortal(); + const site = await serve(root, portal); + try { + expect((await site.ask({ key: 'item:1' })).note).to.equal(SNAPSHOT_VALUE); + expect(await site.ask({ op: 'set', key: 'new', value: LOCAL_WRITE })).to.not.equal(null); + expect(await site.ask({ key: 'new' })).to.equal(LOCAL_WRITE); + expect(await site.ask({ key: 'api-key' })).to.equal('[ENCRYPTED]'); + expect(await site.ask({ key: 'short' })).to.equal('lives'); + expect(await site.ask({ op: 'count', color: 'red' })).to.equal(2); + expect(await site.ask({ op: 'list', color: 'red' })).to.deep.equal(['item:1', 'item:2']); + expect(await site.ask({ key: 'seeded' })).to.equal(null); + expect(await site.ask({ key: 'fixture' })).to.equal(null); + expect(await site.ask({ store: 'local', key: 'own' })).to.equal('mine'); + } finally { + await site.close(); + } + expect(site.output).to.include('KV store sessions: copied 5 keys from the portal (1 encrypted values masked).'); + expect(site.output.some((line) => line.includes('sessions → acme-site-sessions in memory'))).to.equal(true); + expect(site.output.join('\n')).to.not.include('Ledgers of sessions'); + expect(portal.calls.every((call) => call.auth === `Bearer ${TOKEN}`)).to.equal(true); + const printed = site.output.join('\n'); + expect(printed).to.not.include(TOKEN); + expect(fs.existsSync(path.join(root, '.quant'))).to.equal(true); + for (const file of filesUnder(root)) { + const text = fs.readFileSync(file, 'latin1'); + for (const value of [SNAPSHOT_VALUE, LOCAL_WRITE, TOKEN]) expect(text, `${value} in ${file}`).to.not.include(value); + } + const state = JSON.parse(fs.readFileSync(path.join(root, '.quant', 'kv', 'state.json'), 'utf8')); + expect(state.seeded[SNAPSHOT_STORE]).to.equal(undefined); + expect(state.seeded['acme-site-local']).to.be.a('string'); + }); + + it('reads the snapshot again on a restart, and the local write is gone', async () => { + const root = projectRoot(tmp); + const portal = fakePortal(); + const first = await serve(root, portal); + try { + await first.ask({ op: 'set', key: 'item:1', value: LOCAL_WRITE }); + await first.ask({ store: 'local', op: 'set', key: 'kept', value: 'on disk' }); + expect(await first.ask({ key: 'item:1' })).to.equal(LOCAL_WRITE); + } finally { + await first.close(); + } + const reads = portal.calls.length; + const second = await serve(root, portal); + try { + expect(portal.calls.length).to.be.greaterThan(reads); + expect((await second.ask({ key: 'item:1' })).note).to.equal(SNAPSHOT_VALUE); + expect(await second.ask({ store: 'local', key: 'kept' })).to.equal('on disk'); + } finally { + await second.close(); + } + }); + + it('keeps the local state of the store for a session without --kv-remote', async () => { + const root = projectRoot(tmp); + const portal = fakePortal(); + const plain = await serve(root, portal, { 'kv-remote': undefined }); + try { + expect(await plain.ask({ key: 'seeded' })).to.equal('from meta'); + await plain.ask({ op: 'set', key: 'mine', value: 'local only' }); + } finally { + await plain.close(); + } + expect(portal.calls).to.have.length(0); + const remote = await serve(root, portal); + try { + expect(await remote.ask({ key: 'mine' })).to.equal(null); + await remote.ask({ op: 'set', key: 'mine', value: LOCAL_WRITE }); + } finally { + await remote.close(); + } + const again = await serve(root, portal, { 'kv-remote': undefined }); + try { + expect(await again.ask({ key: 'mine' })).to.equal('local only'); + expect(await again.ask({ key: 'item:1' })).to.equal(null); + } finally { + await again.close(); + } + }); + + it('keeps the snapshot and its local writes on a --watch reload, with no new read', async () => { + const root = projectRoot(tmp); + const portal = fakePortal(); + const site = await serve(root, portal, { watch: true }); + try { + await site.ask({ op: 'set', key: 'new', value: 'written' }); + const reads = portal.calls.length; + await new Promise((resolve) => setTimeout(resolve, 300)); + fs.writeFileSync(path.join(root, 'quant', 'edge-functions', 'kv.js'), FUNCTION.replace("'x-version': 'one'", "'x-version': 'two'")); + await until(async () => (await fetch(`${site.url}/kv?key=new`)).headers.get('x-version') === 'two'); + expect(await site.ask({ key: 'new' })).to.equal('written'); + expect((await site.ask({ key: 'item:1' })).note).to.equal(SNAPSHOT_VALUE); + expect(portal.calls.length).to.equal(reads); + } finally { + await site.close(); + } + }); + + it('works with --memory', async () => { + const root = projectRoot(tmp); + const site = await serve(root, fakePortal(), { memory: true }); + try { + expect((await site.ask({ key: 'item:1' })).note).to.equal(SNAPSHOT_VALUE); + expect(await site.ask({ store: 'local', key: 'own' })).to.equal('mine'); + } finally { + await site.close(); + } + expect(fs.existsSync(path.join(root, '.quant'))).to.equal(false); + }); + + it('copies the first 10000 keys only and says that writes stay local', async () => { + const root = projectRoot(tmp); + const items = Array.from({ length: 10001 }, (_, i) => ({ key: `k${String(i).padStart(5, '0')}`, value: i, encrypted: false })); + const site = await serve(root, fakePortal({ items })); + try { + expect(await site.ask({ key: 'k09999' })).to.equal(9999); + expect(await site.ask({ key: 'k10000' })).to.equal(null); + } finally { + await site.close(); + } + expect(site.output).to.include('KV store sessions: copied the first 10000 keys only (limit 10000). Writes stay local.'); + }); + + it('says that ledgers are not copied when quant/kv-ledgers.json has entries for the store', async () => { + const root = projectRoot(tmp, { ledgers: { sessions: [{ name: 'events', collection: 'events', chains: 1, retentionDays: 30 }] } }); + const site = await serve(root, fakePortal()); + await site.close(); + expect(site.output).to.include('Ledgers of sessions are not copied from the portal; declare local ledgers in quant/kv-ledgers.json.'); + }); + + it('refuses a store that quant/meta.json does not declare, before any portal call', async () => { + const root = projectRoot(tmp); + const portal = fakePortal(); + const error = await failure(serve(root, portal, { 'kv-remote': 'other' })); + expect(error.message).to.equal('--kv-remote other is not a KV store in quant/meta.json.'); + expect(portal.calls).to.have.length(0); + }); + + it('stops start-up with the one-line portal error, without the token', async () => { + const root = projectRoot(tmp); + const denied = await failure(serve(root, fakePortal({ status: { '/abc/keys': 403 } }))); + expect(denied.message).to.equal('QUANT_API_TOKEN lacks the kv:read scope for KV store sessions.'); + const missing = await failure(serve(root, fakePortal({ stores: [] }))); + expect(missing.message).to.equal('The KV store sessions is not linked to this project.'); + for (const error of [denied, missing]) expect(`${error.message}\n${error.stack}`).to.not.include(TOKEN); + }); + + it('needs the portal organisation and project', () => { + const root = projectRoot(tmp); + const error = (() => { + try { + resolveOptions({ dir: 'dist', 'kv-remote': 'sessions' }, { root, gitUserName: () => null }); + } catch (err) { + return err; + } + return null; + })(); + expect(error.message).to.equal('--kv-remote needs the portal organisation and project: pass --org and --project, or set them in quant/serve.json or quant.json.'); + }); +}); From 8cdec06ae54d64a13106c97dde63f14f5a823f83 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:52:09 +1000 Subject: [PATCH 21/32] fix(serve): kv-remote review fixes --- README.md | 2 +- src/commands/serve.js | 2 +- src/serve/kv/remote.js | 67 +++++++--- src/serve/kv/wire.js | 3 + src/serve/portal/client.js | 19 ++- src/serve/portal/kv-remote.js | 31 +++-- tests/unit/serve/kv-remote-read.test.mjs | 7 + tests/unit/serve/kv-remote.test.mjs | 158 ++++++++++++++++++++++- tests/unit/serve/portal-client.test.mjs | 11 ++ 9 files changed, 260 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 918c314..1a303e6 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ Flags: - `--clock` - `real` or `manual` (default: `real`). With `manual`, the time of the local KV stops at the start time and, in a session, moves only with `POST /_quant_serve/clock`. See [Local ledgers](#local-ledgers). - `--version-info` - Print the versions of the CLI and its KV and test packages, then exit. - `--pull-rules` - Read the rules of the portal project one time at start, and run them with the rules of `quant/edge-rules.json`. See [Portal rules](#portal-rules---pull-rules). -- `--kv-remote ` - Read the KV store `` of the portal project one time at start (at most 10,000 keys) and serve it from memory. Writes stay local for the session, and a restart reads the portal again. The snapshot is never written to `.quant/`. Secret values read `[ENCRYPTED]`. The store must be in `quant/meta.json`, and the command needs `QUANT_API_TOKEN` and `QUANT_BASE_URL`. +- `--kv-remote ` - Read the KV store `` of the portal project one time at start (at most 10,000 keys) and serve it from memory. Writes stay local for the session, and a restart reads the portal again. The snapshot is never written to `.quant/`. Secret values read `[ENCRYPTED]`. The store must be in `quant/meta.json`. The command needs `QUANT_API_TOKEN` and `QUANT_BASE_URL`, and the organisation and project must be known: from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. A key that `initial_values` reveals once is shown in a box when the store is first seeded. Save the key then. Quant serve also writes it to a file under `.quant/kv/`. diff --git a/src/commands/serve.js b/src/commands/serve.js index f44e735..7dc57a0 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -485,7 +485,7 @@ async function localKv(options, { log, print, portal }) { // environment; the org and project are the serve identity. async function pulledProject(options, { env = process.env, fetchImpl } = {}) { if (!options.pullRules) return null; - const { customer, project, token, base } = portalOptions({ clientid: options.org, project: options.project }, env); + const { customer, project, token, base } = portalOptions({ clientid: options.org, project: options.project }, env, '--pull-rules'); const client = createPortalClient({ base, token, ...(fetchImpl ? { fetchImpl } : {}) }); return pullProject(client, { customer, project }); } diff --git a/src/serve/kv/remote.js b/src/serve/kv/remote.js index d6082b9..ed60577 100644 --- a/src/serve/kv/remote.js +++ b/src/serve/kv/remote.js @@ -10,6 +10,7 @@ import { createPortalClient, portalOptions } from '../portal/client.js'; import { findStore, readStore } from '../portal/kv-remote.js'; export const REMOTE_MAX_KEYS = 10000; +export const REMOTE_MAX_REQUESTS = 200; const MASKED = '[ENCRYPTED]'; // Portal index states that do not serve queries: they are not declared. const UNUSED_STATES = ['FAILED', 'DROPPING']; @@ -20,6 +21,7 @@ const UNUSED_STATES = ['FAILED', 'DROPPING']; * error, conflicts, job) are dropped. */ export function toLocalIndex(def) { + if (def === null || typeof def !== 'object' || Array.isArray(def)) return null; const out = { name: def.name, collection: def.collection, @@ -37,14 +39,14 @@ export function toLocalIndex(def) { * A PortalError stops the start-up with its one-line message. */ export async function readRemoteStore({ name, org, project }, { env = process.env, fetchImpl } = {}) { - const ids = portalOptions({ clientid: org, project }, env); + const ids = portalOptions({ clientid: org, project }, env, '--kv-remote'); const counter = { count: 0 }; const client = createPortalClient({ base: ids.base, token: ids.token, onRequest: () => { counter.count += 1; }, ...(fetchImpl ? { fetchImpl } : {}) }); const ctx = { customer: ids.customer, project: ids.project, name }; const store = await findStore(client, ctx); - const snapshot = await readStore(client, { ...ctx, counter }, store.id, { maxKeys: REMOTE_MAX_KEYS }); + const snapshot = await readStore(client, { ...ctx, counter }, store.id, { maxKeys: REMOTE_MAX_KEYS, maxRequests: REMOTE_MAX_REQUESTS }); return { name, format: store.format ?? 2, ...snapshot }; } @@ -57,12 +59,16 @@ function putOptions(metadata, nowMs) { return ttl > 0 ? { ...options, ttl } : null; } -// A ledger entry, or an entry with no value, is not copied. -const copyable = (entry) => entry.value !== undefined && entry.metadata?.ledger === undefined; +const isLedgerRow = (entry) => entry.metadata?.ledger !== undefined; +// A ledger entry (counted), an entry with no value, or an expired entry is not copied. async function writeEntry(backend, id, entry, result) { + if (isLedgerRow(entry)) { + result.ledgerRows += 1; + return; + } const options = putOptions(entry.metadata || {}, backend.now()); - if (!copyable(entry) || options === null) return; + if (entry.value === undefined || options === null) return; try { await backend.put(id, entry.key, JSON.stringify(entry.value), options); result.copied += 1; @@ -72,14 +78,20 @@ async function writeEntry(backend, id, entry, result) { } } +const notDeclared = (store, index, code) => `KV store ${store}: index ${index} from the portal could not be declared locally (${code}).`; + async function declareIndexes(backend, id, { name, indexes }, { building, warnings }) { for (const def of indexes.filter((item) => !UNUSED_STATES.includes(String(item?.state).toUpperCase()))) { const local = toLocalIndex(def); + if (!local) { + warnings.push(notDeclared(name, '?', 'VALIDATION')); + continue; + } try { await backend.declareIndex(id, local); if (building) await backend.setIndexState(id, local.name, 'BUILDING'); } catch (err) { - warnings.push(`KV store ${name}: index ${local.name} from the portal could not be declared locally (${err.code || err.message}).`); + warnings.push(notDeclared(name, local.name, err.code || err.message)); } } } @@ -89,7 +101,7 @@ async function declareIndexes(backend, id, { name, indexes }, { building, warnin * backend in memory that starts from `base` (the keys and clock offset of * the local backend) with no stores: the index definitions first, then each * entry with its metadata (secret, ttl). - * @returns {Promise<{ data: Object, copied: number, masked: number, warnings: string[] }>} + * @returns {Promise<{ data: Object, copied: number, masked: number, ledgerRows: number, warnings: string[] }>} */ export async function buildRemoteStore({ base, id, snapshot, backendOptions, building = false }) { const { backend } = createLocalKv({ ...backendOptions, storage: undefined }); @@ -97,20 +109,32 @@ export async function buildRemoteStore({ base, id, snapshot, backendOptions, bui await backend.createStore({ id, format: snapshot.format }); const warnings = []; await declareIndexes(backend, id, snapshot, { building, warnings }); - const result = { copied: 0, masked: 0, failed: new Set() }; + const result = { copied: 0, masked: 0, ledgerRows: 0, failed: new Set() }; for (const entry of snapshot.entries) await writeEntry(backend, id, entry, result); if (result.failed.size > 0) { warnings.push(`KV store ${snapshot.name}: some keys from the portal could not be copied locally (${[...result.failed].join(', ')}).`); } const data = backend.snapshot().stores.find((store) => store.id === id); - return { data, copied: result.copied, masked: result.masked, warnings }; + return { data, copied: result.copied, masked: result.masked, ledgerRows: result.ledgerRows, warnings }; +} + +// The cap that stopped a truncated read, in the start-up line. +const STOPPED = { + keys: `limit ${REMOTE_MAX_KEYS}`, + requests: `request limit ${REMOTE_MAX_REQUESTS}`, + cursor: 'the portal ended the list early' +}; + +function copiedLine({ name, truncated, stoppedBy }, { copied, masked, ledgerRows = 0 }) { + const ledger = ledgerRows > 0 ? ` (${ledgerRows} ledger entries not copied)` : ''; + if (!truncated) return `KV store ${name}: copied ${copied} keys from the portal (${masked} encrypted values masked)${ledger}.`; + return `KV store ${name}: copied the first ${copied} keys only (${STOPPED[stoppedBy] || STOPPED.keys})${ledger}. Writes stay local.`; } /** The start-up lines of the remote store. */ -export function remoteLines({ name, truncated }, { copied, masked }, ledgers) { - const lines = [truncated - ? `KV store ${name}: copied the first ${copied} keys only (limit ${REMOTE_MAX_KEYS}). Writes stay local.` - : `KV store ${name}: copied ${copied} keys from the portal (${masked} encrypted values masked).`]; +export function remoteLines(snapshot, built, ledgers) { + const { name } = snapshot; + const lines = [copiedLine(snapshot, built)]; const declared = ledgers.some((entry) => entry.store === name && entry.defs.length > 0); if (declared) lines.push(`Ledgers of ${name} are not copied from the portal; declare local ledgers in quant/kv-ledgers.json.`); return lines; @@ -137,12 +161,17 @@ export function remoteOverlay(storage, id) { storage: { load: () => withRemote(storage.load()), changed: () => storage.changed(), - transaction: (run) => storage.transaction(async (fresh) => { - const disk = fresh ? fresh.stores.find((store) => store.id === id) ?? null : null; - const next = await run(withRemote(fresh)); - if (remote) remote = next.stores.find((store) => store.id === id) ?? remote; - return splice(next, id, disk); - }) + // The new remote state is kept only after the save: a failed save rolls it back too. + transaction: async (run) => { + let after = remote; + await storage.transaction(async (fresh) => { + const disk = fresh ? fresh.stores.find((store) => store.id === id) ?? null : null; + const next = await run(withRemote(fresh)); + if (remote) after = next.stores.find((store) => store.id === id) ?? remote; + return splice(next, id, disk); + }); + remote = after; + } }, setRemote: (data) => { remote = data; } }; diff --git a/src/serve/kv/wire.js b/src/serve/kv/wire.js index 003a291..95d7700 100644 --- a/src/serve/kv/wire.js +++ b/src/serve/kv/wire.js @@ -112,6 +112,8 @@ async function seedInMemory(ctx, seedOptions) { } function seedAll(ctx, seedOptions) { + // The seed copies the disk state (no remote store) and allows it: it must run before loadRemote. + if (ctx.remoteLoaded) return Promise.reject(new Error('The --kv-remote store is loaded: the seed must run before it.')); return ctx.paths ? seedOnFile(ctx, seedOptions) : seedInMemory(ctx, seedOptions); } @@ -205,6 +207,7 @@ async function loadRemote(ctx, snapshot) { backendOptions: backendOptions(null, undefined, ctx.options), building: ctx.indexState === 'BUILDING' }); if (ctx.overlay) ctx.overlay.setRemote(built.data); + ctx.remoteLoaded = true; ctx.backend.restore(withStore(ctx.backend.snapshot(), built.data)); ctx.allowed.set(id, definition); await declareLedgers(ctx.backend, { diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js index 652495d..719edba 100644 --- a/src/serve/portal/client.js +++ b/src/serve/portal/client.js @@ -30,26 +30,35 @@ function savedProject() { } } -function assertPortalUrl(url) { +// The start of an error message: "Rules require", or " needs" (for example --kv-remote). +const subject = (feature) => (feature ? `${feature} needs` : 'Rules require'); + +function assertPortalUrl(url, feature) { const plainLocal = url.protocol === 'http:' && LOCAL_HOSTS.includes(url.hostname); if (url.username || url.password || url.search || url.hash || !/\/api\/v2\/?$/.test(url.pathname) || (url.protocol !== 'https:' && !plainLocal)) { - throw new Error('Rules require an HTTPS portal /api/v2 URL (HTTP is allowed only for localhost)'); + throw new Error(`${subject(feature)} an HTTPS portal /api/v2 URL (HTTP is allowed only for localhost)`); } } +function missingMessage(feature) { + if (!feature) return 'Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'; + return `${feature} needs QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL), and an org and project.`; +} + const first = (...values) => values.find(Boolean); -export function portalOptions(args, env = process.env) { +// feature: the flag in use (for example --kv-remote), named in the errors. Without it, the rules messages. +export function portalOptions(args, env = process.env, feature = undefined) { const saved = savedProject(); const customer = first(args.clientid, args.c, env.QUANT_CLIENT_ID, env.QUANT_CUSTOMER, saved.clientid); const project = first(args.project, args.p, env.QUANT_PROJECT, saved.project); // Deliberately never read QUANT_TOKEN, -t, saved.token, or upload endpoint. // The serve command has no token flag: only the environment is read. const { QUANT_API_TOKEN: token, QUANT_BASE_URL: base } = env; - if (![customer, project, token, base].every(Boolean)) throw new Error('Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'); + if (![customer, project, token, base].every(Boolean)) throw new Error(missingMessage(feature)); const url = new URL(base); - assertPortalUrl(url); + assertPortalUrl(url, feature); return { customer, project, token, base: url }; } diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js index 5067221..8a61c35 100644 --- a/src/serve/portal/kv-remote.js +++ b/src/serve/portal/kv-remote.js @@ -67,17 +67,24 @@ function readKeysPage(client, state, ctx, id, cursor) { const pageKeys = page => (Array.isArray(page?.data) ? page.data : []).map(row => row?.key).filter(key => typeof key === 'string'); -// Reads one page of keys and their values. Returns the next cursor, or a stop flag. +// Why a page stops the read: 'requests' (the request cap cut the values), +// 'keys' (maxKeys), 'cursor' (has_more with no cursor: the rest cannot be +// read), or null (the read goes on, or the list has ended). +function stopReason({ complete, room, count, hasMore, more }) { + if (!complete) return 'requests'; + if (room < count || (more && room <= count)) return 'keys'; + return hasMore && !more ? 'cursor' : null; +} + +// Reads one page of keys and their values. Returns the next cursor, or the stop reason. async function readPage(client, state, ctx, id, { cursor, room }) { const page = await readKeysPage(client, state, ctx, id, cursor); const all = pageKeys(page); const values = await readValues(client, state, ctx, id, all.slice(0, room)); const hasMore = page?.has_more === true; const more = hasMore && Boolean(page.next_cursor); - // has_more with no cursor: the rest cannot be read. - const lost = hasMore && !more; - const cut = !values.complete || room < all.length || (more && room <= all.length) || lost; - return { entries: values.entries, next: more ? page.next_cursor : null, cut }; + const stop = stopReason({ complete: values.complete, room, count: all.length, hasMore, more }); + return { entries: values.entries, next: more && !stop ? page.next_cursor : null, stop }; } export async function readStore(client, ctx, id, { maxKeys = 10000, maxRequests = 200 } = {}) { @@ -85,15 +92,17 @@ export async function readStore(client, ctx, id, { maxKeys = 10000, maxRequests const entries = []; const indexes = state.full() ? [] : await readIndexes(client, state, ctx, id); let cursor; - let truncated = false; + let stoppedBy = null; do { - truncated = state.full(); - if (truncated) break; + if (state.full()) { + stoppedBy = 'requests'; + break; + } const page = await readPage(client, state, ctx, id, { cursor, room: maxKeys - entries.length }); entries.push(...page.entries); - truncated = page.cut; - cursor = truncated ? null : page.next; + stoppedBy = page.stop; + cursor = page.next; } while (cursor); const masked = entries.filter(entry => entry.value === MASKED).length; - return { entries, indexes, masked, truncated, requests: state.requests }; + return { entries, indexes, masked, truncated: stoppedBy !== null, stoppedBy, requests: state.requests }; } diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs index 91f0bab..5633c6a 100644 --- a/tests/unit/serve/kv-remote-read.test.mjs +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -87,6 +87,7 @@ describe('remote KV read', () => { expect(pages.map(c => c.query.cursor)).to.deep.equal([undefined, '1000', '2000']); expect(pages[0].query.limit).to.equal('1000'); expect(result.truncated).to.equal(false); + expect(result.stoppedBy).to.equal(null); }); it('batches values at 100 keys per call', async () => { @@ -131,6 +132,7 @@ describe('remote KV read', () => { const result = await readStore(client, CTX, 'abc', { maxKeys: 1500 }); expect(result.entries).to.have.length(1500); expect(result.truncated).to.equal(true); + expect(result.stoppedBy).to.equal('keys'); }); it('stops at maxRequests and sets truncated', async () => { @@ -139,6 +141,7 @@ describe('remote KV read', () => { expect(calls).to.have.length(4); expect(result.requests).to.equal(4); expect(result.truncated).to.equal(true); + expect(result.stoppedBy).to.equal('requests'); }); it('reports not linked on 404 from a store route', async () => { @@ -162,10 +165,12 @@ describe('remote KV read', () => { const cut = await readStore(more.client, CTX, 'abc', { maxKeys: 1000 }); expect(cut.entries).to.have.length(1000); expect(cut.truncated).to.equal(true); + expect(cut.stoppedBy).to.equal('keys'); expect(more.calls.filter(c => c.route === '/abc/keys')).to.have.length(1); const exact = await readStore(portal({ total: 1000 }).client, CTX, 'abc', { maxKeys: 1000 }); expect(exact.entries).to.have.length(1000); expect(exact.truncated).to.equal(false); + expect(exact.stoppedBy).to.equal(null); }); // A fake portal with one fixed keys page: `page` is the keys reply. @@ -187,6 +192,7 @@ describe('remote KV read', () => { const result = await readStore(client, CTX, 'abc'); expect(result.entries).to.have.length(1); expect(result.truncated).to.equal(true); + expect(result.stoppedBy).to.equal('cursor'); expect(calls.filter(route => route === '/abc/keys')).to.have.length(1); } }); @@ -197,6 +203,7 @@ describe('remote KV read', () => { expect(calls).to.have.length(9); expect(result.requests).to.equal(9); expect(result.truncated).to.equal(true); + expect(result.stoppedBy).to.equal('requests'); }); it('counts 429 retries in requests and in the cap through onRequest', async () => { diff --git a/tests/unit/serve/kv-remote.test.mjs b/tests/unit/serve/kv-remote.test.mjs index 2c243bb..e83a2b0 100644 --- a/tests/unit/serve/kv-remote.test.mjs +++ b/tests/unit/serve/kv-remote.test.mjs @@ -2,8 +2,10 @@ import { expect } from 'chai'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { createLocalKv } from '@quantcdn/edge-kv'; import { resolveOptions, runServe } from '../../../src/commands/serve.js'; -import { toLocalIndex } from '../../../src/serve/kv/remote.js'; +import { buildRemoteStore, remoteLines, remoteOverlay, toLocalIndex } from '../../../src/serve/kv/remote.js'; +import { openLocalKv } from '../../../src/serve/kv/wire.js'; const TOKEN = 'remote-token-never-printed'; const ENV = { QUANT_API_TOKEN: TOKEN, QUANT_BASE_URL: 'http://127.0.0.1:8080/api/v2' }; @@ -107,7 +109,7 @@ async function until(check, timeoutMs = 8000) { } // Start quant serve; returns { url, output, ask(query), close() }. -async function serve(root, portal, flags = {}) { +async function serve(root, portal, flags = {}, env = ENV) { const output = []; let stop; const stopped = new Promise((resolve) => { stop = resolve; }); @@ -117,7 +119,7 @@ async function serve(root, portal, flags = {}) { stop: stopped, print: (line) => output.push(line), log: (line) => output.push(line), - env: ENV, + env, fetchImpl: portal.fetchImpl, onListening: (server) => { url = server.url; } }); @@ -303,6 +305,156 @@ describe('serve --kv-remote', function () { for (const error of [denied, missing]) expect(`${error.message}\n${error.stack}`).to.not.include(TOKEN); }); + it('names the feature when QUANT_API_TOKEN or QUANT_BASE_URL is missing', async () => { + const root = projectRoot(tmp); + const remote = await failure(serve(root, fakePortal(), {}, {})); + expect(remote.message).to.equal('--kv-remote needs QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL), and an org and project.'); + const pull = await failure(serve(root, fakePortal(), { 'kv-remote': undefined, 'pull-rules': true }, { QUANT_API_TOKEN: TOKEN })); + expect(pull.message).to.equal('--pull-rules needs QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL), and an org and project.'); + }); + + describe('buildRemoteStore', () => { + const ID = 'acme-site-sessions'; + const base = () => createLocalKv().backend.snapshot(); + const snapshot = (over) => ({ name: 'sessions', format: 2, indexes: [], entries: [], ...over }); + const build = (over) => buildRemoteStore({ base: base(), id: ID, snapshot: snapshot(over), backendOptions: {} }); + const keysOf = (built) => built.data.items.map(([key]) => key).sort(); + + it('skips expired entries and ledger rows, and counts the ledger rows', async () => { + const past = Math.floor(Date.now() / 1000) - 60; + const built = await build({ entries: [ + { key: 'live', value: 1, metadata: {} }, + { key: 'gone', value: 2, metadata: { expires_at: past } }, + { key: 'events:1', value: { a: 1 }, metadata: { ledger: { chain: 0 } } }, + { key: 'events:2', value: { a: 2 }, metadata: { ledger: { chain: 0 } } } + ] }); + expect(keysOf(built)).to.deep.equal(['live']); + expect(built.copied).to.equal(1); + expect(built.ledgerRows).to.equal(2); + }); + + it('warns with the error code only when a key cannot be copied', async () => { + const built = await build({ + indexes: [{ name: 'byEmail', collection: 'user', fields: ['email'], unique: true }], + entries: [ + { key: 'user:secret-key-name-1', value: { email: 'a' }, metadata: {} }, + { key: 'user:secret-key-name-2', value: { email: 'a' }, metadata: {} } + ] + }); + expect(built.copied).to.equal(1); + expect(built.warnings).to.deep.equal(['KV store sessions: some keys from the portal could not be copied locally (UNIQUE_VIOLATION).']); + expect(built.warnings.join('\n')).to.not.include('secret-key-name'); + }); + + it('warns for an index that cannot be declared, and for an index item that is not an object', async () => { + const built = await build({ indexes: [null, 'text', { name: 'bad name', collection: 'item', fields: [] }, INDEXES[0]] }); + expect(built.warnings).to.deep.equal([ + 'KV store sessions: index ? from the portal could not be declared locally (VALIDATION).', + 'KV store sessions: index ? from the portal could not be declared locally (VALIDATION).', + 'KV store sessions: index bad name from the portal could not be declared locally (VALIDATION).' + ]); + expect(built.data.indexes.map(([name]) => name)).to.deep.equal(['byColor']); + }); + }); + + describe('remoteLines', () => { + const built = { copied: 7, masked: 1, ledgerRows: 0 }; + it('names the cap that stopped the read', () => { + expect(remoteLines({ name: 's', truncated: true, stoppedBy: 'keys' }, built, [])).to.deep.equal(['KV store s: copied the first 7 keys only (limit 10000). Writes stay local.']); + expect(remoteLines({ name: 's', truncated: true, stoppedBy: 'requests' }, built, [])).to.deep.equal(['KV store s: copied the first 7 keys only (request limit 200). Writes stay local.']); + expect(remoteLines({ name: 's', truncated: true, stoppedBy: 'cursor' }, built, [])).to.deep.equal(['KV store s: copied the first 7 keys only (the portal ended the list early). Writes stay local.']); + expect(remoteLines({ name: 's', truncated: false, stoppedBy: null }, built, [])).to.deep.equal(['KV store s: copied 7 keys from the portal (1 encrypted values masked).']); + }); + + it('adds the count of ledger entries not copied, only when there are some', () => { + const ledger = { ...built, ledgerRows: 3 }; + expect(remoteLines({ name: 's', truncated: false, stoppedBy: null }, ledger, [])).to.deep.equal(['KV store s: copied 7 keys from the portal (1 encrypted values masked) (3 ledger entries not copied).']); + expect(remoteLines({ name: 's', truncated: true, stoppedBy: 'keys' }, ledger, [])).to.deep.equal(['KV store s: copied the first 7 keys only (limit 10000) (3 ledger entries not copied). Writes stay local.']); + }); + }); + + describe('remoteOverlay', () => { + it('rolls the remote store back when the save fails', async () => { + const disk = { v: 1, stores: [{ id: 'other', items: [] }] }; + let fail = false; + const inner = { + load: () => disk, + changed: () => false, + transaction: async (run) => { + const next = await run(disk); + if (fail) throw new Error('disk full'); + disk.stores = next.stores; + } + }; + const overlay = remoteOverlay(inner, 'remote'); + overlay.setRemote({ id: 'remote', items: [['a', 1]] }); + fail = true; + const write = (doc) => ({ ...doc, stores: doc.stores.map((store) => (store.id === 'remote' ? { ...store, items: [['a', 2]] } : store)) }); + const error = await failure(overlay.storage.transaction(async (fresh) => write(fresh))); + expect(error.message).to.equal('disk full'); + expect(overlay.storage.load().stores.find((store) => store.id === 'remote').items).to.deep.equal([['a', 1]]); + fail = false; + await overlay.storage.transaction(async (fresh) => write(fresh)); + expect(overlay.storage.load().stores.find((store) => store.id === 'remote').items).to.deep.equal([['a', 2]]); + expect(disk.stores.map((store) => store.id)).to.deep.equal(['other']); + }); + }); + + describe('openLocalKv with a remote store', () => { + const STORES = [{ name: 'sessions', indexes: [], initialValues: [] }, { name: 'local', indexes: [], initialValues: [] }]; + const SNAPSHOT_KEY = 'snapshot-key-never-on-disk'; + const open = (root, remote) => openLocalKv({ root, org: 'acme', project: 'site', stores: STORES, ledgers: [], ...(remote ? { remote } : {}) }); + const snapshot = () => ({ name: 'sessions', format: 2, indexes: [], entries: [{ key: SNAPSHOT_KEY, value: SNAPSHOT_VALUE, metadata: {} }] }); + + it('keeps every write of two instances on one state, and no snapshot data on disk', async () => { + const root = path.join(tmp, 'site'); + fs.mkdirSync(root, { recursive: true }); + const plain = await open(root); + await plain.seed(); + await plain.kvStore.open('acme-site-sessions').set('old', 'disk copy'); + const remote = await open(root, 'sessions'); + await remote.seed(); + await remote.loadRemote(snapshot()); + const r = (name) => remote.kvStore.open(`acme-site-${name}`); + const p = (name) => plain.kvStore.open(`acme-site-${name}`); + await Promise.all(Array.from({ length: 10 }, (_, i) => [ + r('sessions').set(`remote-${i}`, LOCAL_WRITE), + r('local').set(`from-remote-${i}`, i), + p('local').set(`from-plain-${i}`, i), + p('sessions').set(`disk-${i}`, i) + ]).flat()); + for (let i = 0; i < 10; i++) { + expect(await r('sessions').get(`remote-${i}`)).to.equal(LOCAL_WRITE); + expect(await r('local').get(`from-plain-${i}`)).to.equal(i); + expect(await p('local').get(`from-remote-${i}`)).to.equal(i); + expect(await p('sessions').get(`disk-${i}`)).to.equal(i); + expect(await p('sessions').get(`remote-${i}`)).to.equal(null); + } + expect(await r('sessions').get(SNAPSHOT_KEY)).to.equal(SNAPSHOT_VALUE); + expect(await p('sessions').get(SNAPSHOT_KEY)).to.equal(null); + expect(await p('sessions').get('old')).to.equal('disk copy'); + const fresh = await open(root); + expect(await fresh.kvStore.open('acme-site-sessions').get('disk-9')).to.equal(9); + expect(await fresh.kvStore.open('acme-site-local').get('from-remote-9')).to.equal(9); + for (const file of filesUnder(root)) { + const text = fs.readFileSync(file, 'latin1'); + for (const value of [SNAPSHOT_VALUE, SNAPSHOT_KEY, LOCAL_WRITE]) expect(text, `${value} in ${file}`).to.not.include(value); + } + }); + + it('refuses a seed after the remote store is loaded', async () => { + for (const memory of [false, true]) { + const root = path.join(tmp, `seed-${memory}`); + fs.mkdirSync(root, { recursive: true }); + const kv = await openLocalKv({ root, org: 'acme', project: 'site', stores: STORES, ledgers: [], remote: 'sessions', memory }); + await kv.seed(); + await kv.loadRemote(snapshot()); + const error = await failure(kv.seed()); + expect(error.message).to.equal('The --kv-remote store is loaded: the seed must run before it.'); + } + }); + }); + it('needs the portal organisation and project', () => { const root = projectRoot(tmp); const error = (() => { diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index c53beeb..d541056 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -67,6 +67,17 @@ describe('portal client', () => { expect(() => portalOptions(args, {})).to.throw('QUANT_API_TOKEN'); }); + it('names the feature in use when a value is missing, and keeps the rules message without one', () => { + expect(() => portalOptions(args, {})).to.throw('Rules require customer, project, QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL)'); + for (const feature of ['--kv-remote', '--pull-rules']) { + const error = (() => { try { portalOptions(args, {}, feature); } catch (err) { return err; } return null; })(); + expect(error.message).to.equal(`${feature} needs QUANT_API_TOKEN and QUANT_BASE_URL (portal /api/v2 URL), and an org and project.`); + } + expect(() => portalOptions({}, env, '--kv-remote')).to.throw('--kv-remote needs QUANT_API_TOKEN'); + expect(() => portalOptions(args, { ...env, QUANT_BASE_URL: 'http://example.com/api/v2' }, '--kv-remote')) + .to.throw('--kv-remote needs an HTTPS portal /api/v2 URL (HTTP is allowed only for localhost)'); + }); + it('refuses plain HTTP to a remote host', () => { expect(() => portalOptions(args, { ...env, QUANT_BASE_URL: 'http://example.com/api/v2' })).to.throw('HTTPS'); }); From 04d099feaf67f157eb67c9088c604840c78e6c34 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:55:25 +1000 Subject: [PATCH 22/32] test(serve): phase 3 end-to-end against a local portal --- tests/fixtures/fake-portal.mjs | 63 +++++++++ tests/fixtures/fake-portal/fixtures.json | 30 ++++ tests/unit/serve/phase3-e2e.test.mjs | 173 +++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 tests/fixtures/fake-portal.mjs create mode 100644 tests/fixtures/fake-portal/fixtures.json create mode 100644 tests/unit/serve/phase3-e2e.test.mjs diff --git a/tests/fixtures/fake-portal.mjs b/tests/fixtures/fake-portal.mjs new file mode 100644 index 0000000..d0ed38b --- /dev/null +++ b/tests/fixtures/fake-portal.mjs @@ -0,0 +1,63 @@ +/** + * A fake portal for the quant serve end-to-end test. It serves the routes + * that --pull-rules and --kv-remote read from JSON fixtures, on 127.0.0.1, + * and records the method, path and Authorization header of each request. + */ + +import fs from 'node:fs'; +import http from 'node:http'; + +const DEFAULT_FIXTURES = new URL('./fake-portal/fixtures.json', import.meta.url); + +export function loadFixtures(file = DEFAULT_FIXTURES) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function readBody(req) { + return new Promise((resolve) => { + const chunks = []; + req.on('data', (chunk) => chunks.push(chunk)); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + }); +} + +// The routes, as { method, path (after /api/v2/organizations//projects/), reply(body) }. +function routes(fixtures) { + const store = fixtures.stores[0]; + const keys = fixtures.items.map((item) => item.key); + const byKey = new Map(fixtures.items.map((item) => [item.key, item])); + return [ + { method: 'GET', path: '/rules', reply: () => fixtures.rules }, + { method: 'GET', path: '/custom-headers', reply: () => fixtures.customHeaders }, + { method: 'GET', path: '/kv', reply: () => fixtures.stores }, + { method: 'GET', path: `/kv/${store.id}/indexes`, reply: () => ({ indexes: fixtures.indexes }) }, + { method: 'GET', path: `/kv/${store.id}/keys`, reply: () => ({ data: keys.map((key) => ({ key })), next_cursor: null, has_more: false }) }, + { method: 'POST', path: `/kv/${store.id}/batch-get`, reply: (body) => ({ items: JSON.parse(body).keys.map((key) => byKey.get(key) ?? null) }) } + ]; +} + +/** + * Start the fake portal. + * @returns {Promise<{ base: string, requests: Object[], close: () => Promise }>} + * base: the value for QUANT_BASE_URL; requests: { method, path, auth } in order. + */ +export async function startFakePortal(fixtures = loadFixtures()) { + const requests = []; + const prefix = `/api/v2/organizations/${fixtures.organization}/projects/${fixtures.project}`; + const table = routes(fixtures); + const server = http.createServer(async (req, res) => { + const url = new URL(req.url, 'http://127.0.0.1'); + requests.push({ method: req.method, path: url.pathname, auth: req.headers.authorization }); + const body = await readBody(req); + const route = table.find((entry) => entry.method === req.method && `${prefix}${entry.path}` === url.pathname); + res.writeHead(route ? 200 : 404, { 'content-type': 'application/json' }); + res.end(JSON.stringify(route ? route.reply(body) : { message: 'not found' })); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + return { + base: `http://127.0.0.1:${port}/api/v2`, + requests, + close: () => new Promise((resolve) => { server.closeAllConnections(); server.close(resolve); }) + }; +} diff --git a/tests/fixtures/fake-portal/fixtures.json b/tests/fixtures/fake-portal/fixtures.json new file mode 100644 index 0000000..7d6b77e --- /dev/null +++ b/tests/fixtures/fake-portal/fixtures.json @@ -0,0 +1,30 @@ +{ + "organization": "acme", + "project": "site", + "rules": [ + { "uuid": "r-redirect", "name": "Old to new", "action": "redirect", "url": ["/old/*"], "method": "any", "weight": 10, "position": 0, "managed": false, "disabled": false, + "action_config": { "to": "/new/$1", "status_code": 301 } }, + { "uuid": "r-header", "name": "Page header", "action": "headers", "url": ["/page.html"], "method": "any", "weight": 20, "position": 1, "managed": false, "disabled": false, + "action_config": { "headers": { "x-pulled-rule": "applied" } } }, + { "uuid": "r-early-pulled", "name": "Early pulled", "action": "custom_response", "url": ["/early"], "method": "any", "weight": 100, "position": 2, "managed": false, "disabled": false, + "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-100" } }, + { "uuid": "r-order-first", "name": "Order header", "action": "headers", "url": ["/order"], "method": "any", "weight": 150, "position": 3, "managed": false, "disabled": false, + "action_config": { "headers": { "x-before-local": "yes" } } }, + { "uuid": "r-late-pulled", "name": "Late pulled", "action": "custom_response", "url": ["/late"], "method": "any", "weight": 150, "position": 4, "managed": false, "disabled": false, + "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-150" } }, + { "uuid": "r-order-last", "name": "Order last", "action": "custom_response", "url": ["/order"], "method": "any", "weight": 300, "position": 5, "managed": false, "disabled": false, + "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-300" } }, + { "uuid": "r-cookie", "name": "Preview cookie", "action": "custom_response", "url": ["/cookie"], "method": "any", "weight": 310, "position": 6, "managed": false, "disabled": false, + "only_with_cookie": "preview", "action_config": { "custom_response_status_code": 200, "custom_response_body": "cookie-rule" } }, + { "uuid": "r-auth", "name": "Private area", "action": "auth", "url": ["/private/*"], "method": "any", "weight": 320, "position": 7, "managed": false, "disabled": false, + "action_config": { "auth_user": "editor", "auth_pass": "basic-auth-secret-e2e" } }, + { "uuid": "r-proxy", "name": "Backend", "action": "proxy", "url": ["/backend/*"], "method": "any", "weight": 330, "position": 8, "managed": false, "disabled": false, + "action_config": { "to": "http://127.0.0.1:1/", "inject_headers": { "x-origin-key": "inject-secret-e2e" } } } + ], + "customHeaders": { "x-global": "from-portal" }, + "stores": [{ "id": "abc", "name": "cache" }], + "indexes": [], + "items": [ + { "key": "greeting", "value": "hello from the portal", "encrypted": false, "version": "V1" } + ] +} diff --git a/tests/unit/serve/phase3-e2e.test.mjs b/tests/unit/serve/phase3-e2e.test.mjs new file mode 100644 index 0000000..333ff28 --- /dev/null +++ b/tests/unit/serve/phase3-e2e.test.mjs @@ -0,0 +1,173 @@ +import { expect } from 'chai'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { startFakePortal, loadFixtures } from '../../fixtures/fake-portal.mjs'; + +const CLI = new URL('../../../cli.js', import.meta.url).pathname; +const TOKEN = 'test-e2e-token'; +const BASIC_PASSWORD = 'basic-auth-secret-e2e'; +const INJECT_SECRET = 'inject-secret-e2e'; +const LOCAL_WRITE = 'local-write-e2e'; +const PORTAL_VALUE = 'hello from the portal'; +const FN = 'aaaa0000-0000-4000-8000-0000000000e2'; + +const FUNCTION = ` +const url = new URL(event.request.url); +const store = kvStore.open('acme-site-cache'); +if (url.searchParams.get('op') === 'roundtrip') { + const before = await store.get('greeting'); + await store.set('written', url.searchParams.get('value')); + return new Response(JSON.stringify({ before, after: await store.get('written') })); +} +return new Response('local-function'); +`; + +function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(value, null, 2)); +} + +function localRule(id, urls, weight) { + return { id, name: id, type: 'function', urls, weight, function_uuid: FN }; +} + +function writeProject(root) { + const write = (name, text) => { + fs.mkdirSync(path.dirname(path.join(root, name)), { recursive: true }); + fs.writeFileSync(path.join(root, name), text); + }; + write('dist/index.html', 'home'); + write('dist/page.html', 'a page'); + write('dist/private/index.html', 'private page'); + write('quant/edge-functions/kv.js', FUNCTION); + writeJson(path.join(root, 'quant', 'edge-functions.json'), [{ id: 'kv', uuid: FN, type: 'function', path: 'quant/edge-functions/kv.js', description: 'kv' }]); + writeJson(path.join(root, 'quant', 'edge-rules.json'), { version: 1, namespace: 'e2e', rules: [ + localRule('kv', ['/kv'], 500), + localRule('early', ['/early'], 50), + localRule('order', ['/order'], 200), + localRule('late', ['/late'], 200) + ] }); + writeJson(path.join(root, 'quant', 'meta.json'), { kv_stores: [{ name: 'cache' }] }); +} + +function filesUnder(dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + return entry.isDirectory() ? filesUnder(full) : [full]; + }); +} + +// Run `quant serve` as the CLI does. Resolves with { url, output(), stop() } once the URL is printed. +function startCli(root, portal) { + const args = [CLI, 'serve', 'dist', '--port', '0', '--org', 'acme', '--project', 'site', '--pull-rules', '--kv-remote', 'cache']; + const env = { ...process.env, QUANT_API_TOKEN: TOKEN, QUANT_BASE_URL: portal.base }; + const child = spawn(process.execPath, args, { cwd: root, env, stdio: ['ignore', 'pipe', 'pipe'] }); + let text = ''; + child.stdout.on('data', (chunk) => { text += chunk; }); + child.stderr.on('data', (chunk) => { text += chunk; }); + const exited = new Promise((resolve) => child.on('exit', resolve)); + const ready = new Promise((resolve, reject) => { + const timer = setInterval(() => { + const found = /quant serve is running at (\S+)/.exec(text); + if (found) { clearInterval(timer); resolve(found[1]); } + if (child.exitCode !== null) { clearInterval(timer); reject(new Error(`quant serve exited:\n${text}`)); } + }, 50); + }); + return ready.then((url) => ({ + url, + output: () => text, + stop: async () => { child.kill('SIGTERM'); await exited; } + })); +} + +describe('quant serve phase 3 end to end', function () { + this.timeout(30000); + let root; + let portal; + let site; + + before(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-phase3-e2e-')); + writeProject(root); + portal = await startFakePortal(loadFixtures()); + site = await startCli(root, portal); + }); + + after(async () => { + if (site) await site.stop(); + if (portal) await portal.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + const get = (route, init = {}) => fetch(`${site.url}${route}`, { redirect: 'manual', ...init }); + + it('runs a pulled redirect with the $1 substitution', async () => { + const res = await get('/old/thing'); + expect(res.status).to.equal(301); + expect(res.headers.get('location')).to.match(/\/new\/thing$/); + }); + + it('runs a pulled header rule on a static file, and the pulled global header', async () => { + const res = await get('/page.html'); + expect(res.status).to.equal(200); + expect(await res.text()).to.equal('a page'); + expect(res.headers.get('x-pulled-rule')).to.equal('applied'); + expect(res.headers.get('x-global')).to.equal('from-portal'); + }); + + it('places a local rule among the pulled rules by weight', async () => { + const order = await get('/order'); + expect(await order.text()).to.equal('local-function'); + expect(order.headers.get('x-before-local')).to.equal('yes'); + expect(await (await get('/early')).text()).to.equal('local-function'); + expect(await (await get('/late')).text()).to.equal('pulled-150'); + }); + + it('runs the cookie rule only with the cookie', async () => { + expect((await get('/cookie')).status).to.equal(404); + const res = await get('/cookie', { headers: { cookie: 'preview=1' } }); + expect(await res.text()).to.equal('cookie-rule'); + }); + + it('keeps pulled secrets out of the responses', async () => { + const denied = await get('/private/index.html'); + expect(denied.status).to.equal(401); + const basic = Buffer.from(`editor:${BASIC_PASSWORD}`).toString('base64'); + const allowed = await get('/private/index.html', { headers: { authorization: `Basic ${basic}` } }); + expect(await allowed.text()).to.equal('private page'); + const failed = await get('/backend/x'); + const body = await failed.text(); + expect(failed.status).to.equal(502); + expect(body).to.not.include(INJECT_SECRET); + expect([...failed.headers.values()].join('\n')).to.not.include(INJECT_SECRET); + }); + + it('reads a remote key, writes a new key and reads it back', async () => { + const res = await get(`/kv?op=roundtrip&value=${LOCAL_WRITE}`); + expect(await res.json()).to.deep.equal({ before: PORTAL_VALUE, after: LOCAL_WRITE }); + }); + + it('reads the portal with GET and batch-get only, each with the bearer token', () => { + const methodPaths = portal.requests.map((request) => `${request.method} ${request.path.split('/').pop()}`); + expect(methodPaths.filter((entry) => entry.startsWith('POST'))).to.deep.equal(['POST batch-get']); + expect(portal.requests.every((request) => request.method === 'GET' || request.path.endsWith('/batch-get'))).to.equal(true); + expect(portal.requests.length).to.be.greaterThan(4); + for (const request of portal.requests) expect(request.auth).to.equal(`Bearer ${TOKEN}`); + }); + + it('prints no token or secret, and writes no snapshot value or token to the project', async () => { + const printed = site.output(); + expect(printed).to.include('Pulled 9 portal rules (4 from quant/edge-rules.json).'); + expect(printed).to.include('KV store cache: copied 1 keys from the portal'); + for (const secret of [TOKEN, BASIC_PASSWORD, INJECT_SECRET, PORTAL_VALUE, LOCAL_WRITE]) expect(printed).to.not.include(secret); + await site.stop(); + expect(fs.existsSync(path.join(root, '.quant'))).to.equal(true); + for (const file of filesUnder(root)) { + const text = fs.readFileSync(file, 'latin1'); + for (const secret of [TOKEN, PORTAL_VALUE, LOCAL_WRITE, BASIC_PASSWORD, INJECT_SECRET]) expect(text, `${secret} in ${file}`).to.not.include(secret); + } + site = null; + }); +}); From bff5ffaf8c22b5e9c45b2c91ea2fb4fe1c307f35 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:56:28 +1000 Subject: [PATCH 23/32] refactor(serve): remove the empty later-release flag gate --- src/commands/serve.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/commands/serve.js b/src/commands/serve.js index 7dc57a0..4e6ae48 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -20,7 +20,6 @@ import { pullProject } from '../serve/portal/pull-rules.js'; import { readRemoteStore, remoteLines } from '../serve/kv/remote.js'; const DEFAULTS = Object.freeze({ dir: 'dist', port: 8787, host: '127.0.0.1', org: 'local', project: 'dev', userName: 'Owner' }); -const LATER_RELEASE = []; const IDENTITY_HELP = 'pass --org and --project, or set them in quant/serve.json or quant.json.'; const PULL_IDENTITY = `--pull-rules needs the portal organisation and project: ${IDENTITY_HELP}`; const REMOTE_IDENTITY = `--kv-remote needs the portal organisation and project: ${IDENTITY_HELP}`; @@ -57,12 +56,6 @@ function readGitUserName(root) { } } -function refuseLaterFlags(args) { - for (const flag of LATER_RELEASE) { - if (args[flag] !== undefined && args[flag] !== false) throw new Error(`--${flag} is available in a later release`); - } -} - function portOf(value) { const port = Number(first([value], DEFAULTS.port)); if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error('--port must be a port number (0-65535)'); @@ -196,7 +189,6 @@ function devOptions(args) { * @param {{ root?: string, gitUserName?: (root: string) => string|null }} [deps] */ export function resolveOptions(args, { root = process.cwd(), gitUserName = readGitUserName } = {}) { - refuseLaterFlags(args); // The org, project and meta defaults of quant/serve.json (a flag wins). const local = readServeConfig(root).defaults; return { @@ -662,7 +654,7 @@ export default { .option('watch', { type: 'boolean', describe: 'Reload on changes; run --build again on changes under src/ and public/' }) .option('run', { type: 'string', describe: 'Long-running command (a dev server) to start beside the server' }) .option('proxy', { type: 'string', describe: 'Dev server URL for requests that no rule ends' }) - .option('pull-rules', { type: 'boolean', describe: 'Read the portal rules at start and run them with the local rules (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) + .option('pull-rules', { type: 'boolean', describe: 'Read the portal rules and global headers once at start and run them with the local rules (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) .option('kv-remote', { type: 'string', describe: 'Serve one portal KV store from a snapshot read at start; writes stay local (needs QUANT_API_TOKEN and QUANT_BASE_URL)' }) .option('version-info', { type: 'boolean', describe: 'Print the versions and exit' }), promptArgs: async () => { From 9e850b09dd1ccb1b733625e9f1716c4421a35a21 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:56:28 +1000 Subject: [PATCH 24/32] docs(serve): portal rules and remote KV --- README.md | 71 ++++++++++++++++++++++++++---------- docs/adopting-quant-serve.md | 9 +++++ 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 1a303e6..0a396c8 100644 --- a/README.md +++ b/README.md @@ -282,8 +282,8 @@ Flags: - `--indexes` - Index state for the session: `active` or `building` (default: `active`). With `building`, the indexes of the stores stay in the building state. - `--clock` - `real` or `manual` (default: `real`). With `manual`, the time of the local KV stops at the start time and, in a session, moves only with `POST /_quant_serve/clock`. See [Local ledgers](#local-ledgers). - `--version-info` - Print the versions of the CLI and its KV and test packages, then exit. -- `--pull-rules` - Read the rules of the portal project one time at start, and run them with the rules of `quant/edge-rules.json`. See [Portal rules](#portal-rules---pull-rules). -- `--kv-remote ` - Read the KV store `` of the portal project one time at start (at most 10,000 keys) and serve it from memory. Writes stay local for the session, and a restart reads the portal again. The snapshot is never written to `.quant/`. Secret values read `[ENCRYPTED]`. The store must be in `quant/meta.json`. The command needs `QUANT_API_TOKEN` and `QUANT_BASE_URL`, and the organisation and project must be known: from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. +- `--pull-rules` - Read the rules and global headers of the portal project once at start, and run them with the rules of `quant/edge-rules.json`. See [Portal rules and remote KV](#portal-rules-and-remote-kv). +- `--kv-remote ` - Read the KV store `` of the portal project once at start (at most 10,000 keys) and serve it from memory. Writes stay local. See [Portal rules and remote KV](#portal-rules-and-remote-kv). A key that `initial_values` reveals once is shown in a box when the store is first seeded. Save the key then. Quant serve also writes it to a file under `.quant/kv/`. @@ -466,29 +466,62 @@ The order is the order of the edge: global headers, basic authentication, rules, `quant/serve.json` is local only. No deploy command reads it, and `quant deploy` never uploads it. `quant serve` never prints the password. Do not put a production password in the file. A file with the wrong shape stops `quant serve` at start-up, with a message that names the field. It also stops `quant kv --local` (exit code 1), because that command reads the `org`, `project` and `meta` defaults from the same file. With `--watch`, `quant serve` reads the file again when it changes. If the new file has the wrong shape, `quant serve` prints the message and keeps the last valid settings. -## Portal rules (--pull-rules) +## Portal rules and remote KV -`quant serve --pull-rules` reads the rules of the portal project one time at start. The rules run locally in the order of the edge, with the rules of `quant/edge-rules.json`. +Two flags read data from the portal project when `quant serve` starts: `--pull-rules` for the rules and global headers, and `--kv-remote ` for one KV store. Use them to run the site locally against the rules and data of a real project. Both flags only read. Neither flag changes anything on the portal. ```bash -export QUANT_API_TOKEN=... # a portal token with the rules:read scope +export QUANT_API_TOKEN=... # a portal token with the scopes below export QUANT_BASE_URL=https://dashboard.quantcdn.io/api/v2 -quant serve dist --pull-rules --org acme --project marketing-site +quant serve dist --pull-rules --kv-remote sessions --org acme --project marketing-site ``` -- The token and the portal URL come only from `QUANT_API_TOKEN` and `QUANT_BASE_URL`. The organisation and the project come from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. `--pull-rules` stops if no organisation or project is set. -- The token needs the `rules:read` scope. Without it, `quant serve` stops with a message that names the scope. -- The rules that a deploy of `quant/edge-rules.json` owns on the portal are replaced by the rules of the local file. Each local rule goes before the first portal rule with a higher weight. This is the order that the edge has after a rules deploy of the same file. -- The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules from `quant/edge-rules.json` that run with the portal rules. -- With `--watch`, a change to `quant/edge-rules.json` merges the local rules again with the portal rules in memory. `quant serve` does not read the portal again. -- A rule type that quant serve does not run (for example a bot challenge) is skipped, with one warning at start-up. A rule with a country, IP, ASN, user agent or traffic source condition is skipped, with one warning. -- A proxy rule with failover mode on or with TLS verification off gets one warning: these settings work differently in quant serve. -- A rule with a cookie condition runs only when the request sends a cookie of exactly that name. -- For some organisations the portal leaves some rule types out of the list. Then `quant serve` prints one warning, and those rules do not run locally. -- With the `projects:read` scope, the global headers of the portal project are used too. The `globalHeaders` of `quant/serve.json` override a portal header of the same name, with one warning for each name. Without the scope, `quant serve` prints one warning and uses only the local headers. -- Basic authentication and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. -- The pulled rules and headers stay in memory. `quant serve` does not write them to `.quant/` and does not print a setting value. -- A pulled function rule runs only when its function uuid is in `quant/edge-functions.json`. +### Token, URL and project + +- `QUANT_API_TOKEN` and `QUANT_BASE_URL` are the only source of the token and the portal URL. There is no flag for either, and `quant serve` never prints the token. +- `QUANT_BASE_URL` must be an HTTPS portal `/api/v2` URL. Plain HTTP is allowed only for `localhost`, `127.0.0.1` and `[::1]`. +- The organisation and project come from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. If none sets them, the command stops with a message that names the flag. +- Scopes: `rules:read` for the rules, `projects:read` for the global headers, and `kv:read` for the KV store. Without `rules:read` or `kv:read`, `quant serve` stops and the message names the scope. Without `projects:read`, `quant serve` prints one warning and uses only the local global headers. + +### What the snapshot is + +- The portal is read once, at start. Nothing is read again while `quant serve` runs. With `--watch`, a reload merges the local files again with the data in memory and makes no portal request. +- The rules, the headers and the KV entries stay in memory. `quant serve` does not write them to `.quant/` or to any other file, and it does not print a rule setting or a KV value. +- Writes to the remote store stay local. They last for the session and they are lost when `quant serve` stops. A restart reads the portal again. + +### Rules (--pull-rules) + +The pulled rules run in the order of the edge, with the rules of `quant/edge-rules.json`. The rules that a deploy of `quant/edge-rules.json` owns on the portal are replaced by the local rules. Each local rule goes before the first portal rule with a higher weight. This is the order that the edge has after a deploy of the same file. + +The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules that run with the portal rules. + +Actions that run locally: redirect (with `$1` to `$5`), headers, custom response, basic auth, error page, proxy, and edge function, auth function and filter rules. A rule with a cookie condition runs only when the request sends a cookie of exactly that name. + +These do not run locally: + +- A rule action that quant serve does not have (for example a bot challenge or serve-static). `quant serve` skips the rule and prints `Rule () is not available in quant serve and is skipped.` +- A rule with a country, IP, ASN, user agent or traffic source condition. `quant serve` skips the rule and prints `Rule : the condition(s) is/are not checked in quant serve, so the rule is skipped.` +- The WAF settings of a proxy rule. `quant serve` prints `Rule : WAF settings are ignored in quant serve.` +- The host setting of a proxy rule. The origin gets the host of the `to` URL, and `quant serve` prints `Rule : the host setting is ignored in quant serve. The origin gets the host of the to URL.` +- The failover mode and TLS verification settings of a proxy rule. `quant serve` prints `Rule : failover and TLS verification settings work differently in quant serve.` + +Other points: + +- Some organisations (for example trial organisations) do not get every rule type in the portal list. `quant serve` then prints `Some rule types are not in the portal response for this organisation, so they do not run locally.`, and the missing rules do not run. +- Basic authentication for the whole site and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. +- The `globalHeaders` of `quant/serve.json` replace a portal global header of the same name, with one warning for each name. +- A pulled function rule runs only when its function uuid is in `quant/edge-functions.json`. The function code is not pulled. + +### Remote KV (--kv-remote) + +`--kv-remote ` copies the KV store `` into the local KV, in memory. The store must be linked to the portal project and listed in `quant/meta.json`, so that functions can open it. The `initial_values` and fixtures of that store are not seeded. + +- Caps: at most 10,000 keys and 200 portal requests. If a cap stops the read, `quant serve` prints `KV store : copied the first keys only (limit 10000). Writes stay local.`. For the request cap, the text in the brackets is `request limit 200`. If the portal ends the key list early, it is `the portal ended the list early`. +- The start-up output shows `KV store : copied keys from the portal ( encrypted values masked).` +- A secret value reads `[ENCRYPTED]`. The portal does not send the real value. +- Index definitions are copied, so index lists and counts work. Expiry times are kept. +- Ledger entries are not copied. If `quant/kv-ledgers.json` declares ledgers for the store, those ledgers run locally and start empty. +- `quant kv --local` in a second process does not see the snapshot. It sees only the state on disk. ## Edge libraries in quant serve diff --git a/docs/adopting-quant-serve.md b/docs/adopting-quant-serve.md index 8a1d517..c7ddda7 100644 --- a/docs/adopting-quant-serve.md +++ b/docs/adopting-quant-serve.md @@ -124,6 +124,15 @@ bundles function sources into the committed function bodies. - `quant serve --indexes building` and `--kv-lag 1000` show the not-ready and list-lag states that production can have, so you can test the messages for them. +## Use the rules and data of the portal project + +To run the site against the rules and a KV store of the real project, start +with `quant serve --pull-rules --kv-remote `. Both flags read the portal +once at start, keep the data in memory and never write it to `.quant/`. They +need `QUANT_API_TOKEN` and `QUANT_BASE_URL`. See +[Portal rules and remote KV](../README.md#portal-rules-and-remote-kv) for the +scopes, the caps and the rule actions that do not run locally. + ## Limits quant serve runs documented edge behaviour. It does not run scheduled From 74e9082f694f0668209020131be049ce4c02350b Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 22:59:08 +1000 Subject: [PATCH 25/32] test(serve): stop the CLI on a stalled start-up; docs fixes --- README.md | 6 +- tests/fixtures/fake-portal/fixtures.json | 205 ++++++++++++++++++++--- tests/unit/serve/phase3-e2e.test.mjs | 29 +++- 3 files changed, 207 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 0a396c8..29234bb 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ Actions that run locally: redirect (with `$1` to `$5`), headers, custom response These do not run locally: - A rule action that quant serve does not have (for example a bot challenge or serve-static). `quant serve` skips the rule and prints `Rule () is not available in quant serve and is skipped.` -- A rule with a country, IP, ASN, user agent or traffic source condition. `quant serve` skips the rule and prints `Rule : the condition(s) is/are not checked in quant serve, so the rule is skipped.` +- A rule with a country, IP, ASN, user agent or traffic source condition. `quant serve` skips the rule and prints `Rule : the condition is not checked in quant serve, so the rule is skipped.` (one condition) or `Rule : the , conditions are not checked in quant serve, so the rule is skipped.` (more than one). - The WAF settings of a proxy rule. `quant serve` prints `Rule : WAF settings are ignored in quant serve.` - The host setting of a proxy rule. The origin gets the host of the `to` URL, and `quant serve` prints `Rule : the host setting is ignored in quant serve. The origin gets the host of the to URL.` - The failover mode and TLS verification settings of a proxy rule. `quant serve` prints `Rule : failover and TLS verification settings work differently in quant serve.` @@ -517,10 +517,10 @@ Other points: `--kv-remote ` copies the KV store `` into the local KV, in memory. The store must be linked to the portal project and listed in `quant/meta.json`, so that functions can open it. The `initial_values` and fixtures of that store are not seeded. - Caps: at most 10,000 keys and 200 portal requests. If a cap stops the read, `quant serve` prints `KV store : copied the first keys only (limit 10000). Writes stay local.`. For the request cap, the text in the brackets is `request limit 200`. If the portal ends the key list early, it is `the portal ended the list early`. -- The start-up output shows `KV store : copied keys from the portal ( encrypted values masked).` +- The start-up output shows `KV store : copied keys from the portal ( encrypted values masked).` If the portal store has ledger entries, the line adds ` ( ledger entries not copied)` before the final full stop. - A secret value reads `[ENCRYPTED]`. The portal does not send the real value. - Index definitions are copied, so index lists and counts work. Expiry times are kept. -- Ledger entries are not copied. If `quant/kv-ledgers.json` declares ledgers for the store, those ledgers run locally and start empty. +- Ledger entries are not copied. If `quant/kv-ledgers.json` declares ledgers for the store, those ledgers run locally and start empty, and `quant serve` prints `Ledgers of are not copied from the portal; declare local ledgers in quant/kv-ledgers.json.` - `quant kv --local` in a second process does not see the snapshot. It sees only the state on disk. ## Edge libraries in quant serve diff --git a/tests/fixtures/fake-portal/fixtures.json b/tests/fixtures/fake-portal/fixtures.json index 7d6b77e..509da75 100644 --- a/tests/fixtures/fake-portal/fixtures.json +++ b/tests/fixtures/fake-portal/fixtures.json @@ -2,29 +2,192 @@ "organization": "acme", "project": "site", "rules": [ - { "uuid": "r-redirect", "name": "Old to new", "action": "redirect", "url": ["/old/*"], "method": "any", "weight": 10, "position": 0, "managed": false, "disabled": false, - "action_config": { "to": "/new/$1", "status_code": 301 } }, - { "uuid": "r-header", "name": "Page header", "action": "headers", "url": ["/page.html"], "method": "any", "weight": 20, "position": 1, "managed": false, "disabled": false, - "action_config": { "headers": { "x-pulled-rule": "applied" } } }, - { "uuid": "r-early-pulled", "name": "Early pulled", "action": "custom_response", "url": ["/early"], "method": "any", "weight": 100, "position": 2, "managed": false, "disabled": false, - "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-100" } }, - { "uuid": "r-order-first", "name": "Order header", "action": "headers", "url": ["/order"], "method": "any", "weight": 150, "position": 3, "managed": false, "disabled": false, - "action_config": { "headers": { "x-before-local": "yes" } } }, - { "uuid": "r-late-pulled", "name": "Late pulled", "action": "custom_response", "url": ["/late"], "method": "any", "weight": 150, "position": 4, "managed": false, "disabled": false, - "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-150" } }, - { "uuid": "r-order-last", "name": "Order last", "action": "custom_response", "url": ["/order"], "method": "any", "weight": 300, "position": 5, "managed": false, "disabled": false, - "action_config": { "custom_response_status_code": 200, "custom_response_body": "pulled-300" } }, - { "uuid": "r-cookie", "name": "Preview cookie", "action": "custom_response", "url": ["/cookie"], "method": "any", "weight": 310, "position": 6, "managed": false, "disabled": false, - "only_with_cookie": "preview", "action_config": { "custom_response_status_code": 200, "custom_response_body": "cookie-rule" } }, - { "uuid": "r-auth", "name": "Private area", "action": "auth", "url": ["/private/*"], "method": "any", "weight": 320, "position": 7, "managed": false, "disabled": false, - "action_config": { "auth_user": "editor", "auth_pass": "basic-auth-secret-e2e" } }, - { "uuid": "r-proxy", "name": "Backend", "action": "proxy", "url": ["/backend/*"], "method": "any", "weight": 330, "position": 8, "managed": false, "disabled": false, - "action_config": { "to": "http://127.0.0.1:1/", "inject_headers": { "x-origin-key": "inject-secret-e2e" } } } + { + "uuid": "r-redirect", + "name": "Old to new", + "action": "redirect", + "url": [ + "/old/*" + ], + "method": "any", + "weight": 10, + "position": 0, + "managed": false, + "disabled": false, + "action_config": { + "to": "/new/$1", + "status_code": 301 + } + }, + { + "uuid": "r-header", + "name": "Page header", + "action": "headers", + "url": [ + "/page.html" + ], + "method": "any", + "weight": 20, + "position": 1, + "managed": false, + "disabled": false, + "action_config": { + "headers": { + "x-pulled-rule": "applied" + } + } + }, + { + "uuid": "r-early-pulled", + "name": "Early pulled", + "action": "custom_response", + "url": [ + "/early" + ], + "method": "any", + "weight": 100, + "position": 2, + "managed": false, + "disabled": false, + "action_config": { + "custom_response_status_code": 200, + "custom_response_body": "pulled-100" + } + }, + { + "uuid": "r-order-first", + "name": "Order header", + "action": "headers", + "url": [ + "/order" + ], + "method": "any", + "weight": 150, + "position": 3, + "managed": false, + "disabled": false, + "action_config": { + "headers": { + "x-before-local": "yes" + } + } + }, + { + "uuid": "r-late-pulled", + "name": "Late pulled", + "action": "custom_response", + "url": [ + "/late" + ], + "method": "any", + "weight": 150, + "position": 4, + "managed": false, + "disabled": false, + "action_config": { + "custom_response_status_code": 200, + "custom_response_body": "pulled-150" + } + }, + { + "uuid": "r-order-last", + "name": "Order last", + "action": "custom_response", + "url": [ + "/order" + ], + "method": "any", + "weight": 300, + "position": 5, + "managed": false, + "disabled": false, + "action_config": { + "custom_response_status_code": 200, + "custom_response_body": "pulled-300" + } + }, + { + "uuid": "r-cookie", + "name": "Preview cookie", + "action": "custom_response", + "url": [ + "/cookie" + ], + "method": "any", + "weight": 310, + "position": 6, + "managed": false, + "disabled": false, + "only_with_cookie": "preview", + "action_config": { + "custom_response_status_code": 200, + "custom_response_body": "cookie-rule" + } + }, + { + "uuid": "r-auth", + "name": "Private area", + "action": "auth", + "url": [ + "/private/*" + ], + "method": "any", + "weight": 320, + "position": 7, + "managed": false, + "disabled": false, + "action_config": { + "auth_user": "editor", + "auth_pass": "basic-auth-secret-e2e" + } + }, + { + "uuid": "r-proxy", + "name": "Backend", + "action": "proxy", + "url": [ + "/backend/*" + ], + "method": "any", + "weight": 330, + "position": 8, + "managed": false, + "disabled": false, + "action_config": { + "to": "http://127.0.0.1:1/", + "inject_headers": { + "x-origin-key": "inject-secret-e2e" + } + } + } + ], + "customHeaders": { + "x-global": "from-portal" + }, + "stores": [ + { + "id": "abc", + "name": "cache" + } ], - "customHeaders": { "x-global": "from-portal" }, - "stores": [{ "id": "abc", "name": "cache" }], "indexes": [], "items": [ - { "key": "greeting", "value": "hello from the portal", "encrypted": false, "version": "V1" } + { + "key": "greeting", + "value": "hello from the portal", + "encrypted": false, + "version": "V1" + }, + { + "key": "events:1", + "value": { + "a": 1 + }, + "encrypted": false, + "version": "V1", + "ledger": { + "chain": 0 + } + } ] } diff --git a/tests/unit/serve/phase3-e2e.test.mjs b/tests/unit/serve/phase3-e2e.test.mjs index 333ff28..0688ce1 100644 --- a/tests/unit/serve/phase3-e2e.test.mjs +++ b/tests/unit/serve/phase3-e2e.test.mjs @@ -11,6 +11,7 @@ const BASIC_PASSWORD = 'basic-auth-secret-e2e'; const INJECT_SECRET = 'inject-secret-e2e'; const LOCAL_WRITE = 'local-write-e2e'; const PORTAL_VALUE = 'hello from the portal'; +const START_DEADLINE_MS = 20000; const FN = 'aaaa0000-0000-4000-8000-0000000000e2'; const FUNCTION = ` @@ -21,6 +22,11 @@ if (url.searchParams.get('op') === 'roundtrip') { await store.set('written', url.searchParams.get('value')); return new Response(JSON.stringify({ before, after: await store.get('written') })); } +if (url.searchParams.get('op') === 'ledger') { + const rows = []; + for await (const item of store.list({ prefix: 'events:' })) rows.push(item.key); + return new Response(JSON.stringify(rows)); +} return new Response('local-function'); `; @@ -49,6 +55,7 @@ function writeProject(root) { localRule('order', ['/order'], 200), localRule('late', ['/late'], 200) ] }); + writeJson(path.join(root, 'quant', 'kv-ledgers.json'), { cache: [{ name: 'events', collection: 'events', chains: 1, retentionDays: 30 }] }); writeJson(path.join(root, 'quant', 'meta.json'), { kv_stores: [{ name: 'cache' }] }); } @@ -69,17 +76,16 @@ function startCli(root, portal) { child.stderr.on('data', (chunk) => { text += chunk; }); const exited = new Promise((resolve) => child.on('exit', resolve)); const ready = new Promise((resolve, reject) => { + const started = Date.now(); const timer = setInterval(() => { const found = /quant serve is running at (\S+)/.exec(text); if (found) { clearInterval(timer); resolve(found[1]); } - if (child.exitCode !== null) { clearInterval(timer); reject(new Error(`quant serve exited:\n${text}`)); } + else if (child.exitCode !== null) { clearInterval(timer); reject(new Error(`quant serve exited:\n${text}`)); } + else if (Date.now() - started > START_DEADLINE_MS) { clearInterval(timer); reject(new Error(`quant serve did not start in time:\n${text}`)); } }, 50); }); - return ready.then((url) => ({ - url, - output: () => text, - stop: async () => { child.kill('SIGTERM'); await exited; } - })); + const stop = async () => { child.kill('SIGTERM'); await exited; }; + return { ready, output: () => text, stop }; } describe('quant serve phase 3 end to end', function () { @@ -92,7 +98,8 @@ describe('quant serve phase 3 end to end', function () { root = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-phase3-e2e-')); writeProject(root); portal = await startFakePortal(loadFixtures()); - site = await startCli(root, portal); + site = startCli(root, portal); + site.url = await site.ready; }); after(async () => { @@ -149,6 +156,12 @@ describe('quant serve phase 3 end to end', function () { expect(await res.json()).to.deep.equal({ before: PORTAL_VALUE, after: LOCAL_WRITE }); }); + it('starts a ledger declared for the remote store empty', async () => { + expect(await (await get('/kv?op=ledger')).json()).to.deep.equal([]); + expect(site.output()).to.include('KV store cache: copied 1 keys from the portal (0 encrypted values masked) (1 ledger entries not copied).'); + expect(site.output()).to.include('Ledgers of cache are not copied from the portal; declare local ledgers in quant/kv-ledgers.json.'); + }); + it('reads the portal with GET and batch-get only, each with the bearer token', () => { const methodPaths = portal.requests.map((request) => `${request.method} ${request.path.split('/').pop()}`); expect(methodPaths.filter((entry) => entry.startsWith('POST'))).to.deep.equal(['POST batch-get']); @@ -160,7 +173,6 @@ describe('quant serve phase 3 end to end', function () { it('prints no token or secret, and writes no snapshot value or token to the project', async () => { const printed = site.output(); expect(printed).to.include('Pulled 9 portal rules (4 from quant/edge-rules.json).'); - expect(printed).to.include('KV store cache: copied 1 keys from the portal'); for (const secret of [TOKEN, BASIC_PASSWORD, INJECT_SECRET, PORTAL_VALUE, LOCAL_WRITE]) expect(printed).to.not.include(secret); await site.stop(); expect(fs.existsSync(path.join(root, '.quant'))).to.equal(true); @@ -168,6 +180,5 @@ describe('quant serve phase 3 end to end', function () { const text = fs.readFileSync(file, 'latin1'); for (const secret of [TOKEN, PORTAL_VALUE, LOCAL_WRITE, BASIC_PASSWORD, INJECT_SECRET]) expect(text, `${secret} in ${file}`).to.not.include(secret); } - site = null; }); }); From e848cfe76318254031604d4e1f67b5f7e556f04f Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 23:13:50 +1000 Subject: [PATCH 26/32] fix(serve): keep pulled managed rules that the local file does not replace A rules deploy creates or updates only the manifest rules of its own namespace and never removes a rule. The merge now drops only a pulled managed rule whose deploy namespace and id match a rule of quant/edge-rules.json, and keeps every other pulled rule in its order. --- README.md | 2 +- src/serve/portal/pull-rules.js | 28 +++++++--- src/serve/project.js | 7 +-- tests/unit/serve/pull-rules.test.mjs | 76 ++++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 29234bb..be67556 100644 --- a/README.md +++ b/README.md @@ -491,7 +491,7 @@ quant serve dist --pull-rules --kv-remote sessions --org acme --project marketin ### Rules (--pull-rules) -The pulled rules run in the order of the edge, with the rules of `quant/edge-rules.json`. The rules that a deploy of `quant/edge-rules.json` owns on the portal are replaced by the local rules. Each local rule goes before the first portal rule with a higher weight. This is the order that the edge has after a deploy of the same file. +The pulled rules run in the order of the edge, with the rules of `quant/edge-rules.json`. A local rule replaces the portal rule that a deploy of the same namespace and rule ID owns. All other portal rules stay in their order: also a deployed rule that the local file no longer lists, and the deployed rules of other namespaces. Each local rule then goes before the first portal rule with a higher weight. A deploy also creates or updates only the rules of its own namespace and removes no rule, so this is the order that the edge has after a deploy of the same file when the portal rules are in weight order. Without `quant/edge-rules.json`, the portal rules run unchanged. The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules that run with the portal rules. diff --git a/src/serve/portal/pull-rules.js b/src/serve/portal/pull-rules.js index cc8ed92..542074c 100644 --- a/src/serve/portal/pull-rules.js +++ b/src/serve/portal/pull-rules.js @@ -115,6 +115,14 @@ function ruleId(item) { return String(item.uuid ?? item.rule_id ?? `position-${item.position}`); } +// The manifest namespace and rule id of a rule that a rules deploy owns +// (managed), from its deployment marker. null for any other rule. +function deploymentOf(item) { + const marker = item.managed === true && isObject(item.deployment) ? item.deployment : null; + const valid = marker && typeof marker.namespace === 'string' && typeof marker.id === 'string'; + return valid ? { namespace: marker.namespace, id: marker.id } : null; +} + function domains(value) { const list = textList(value).map((domain) => domain.toLowerCase()); return list.length > 0 ? list : ['any']; @@ -139,20 +147,28 @@ export function fromPortalRule(item) { disabled: toBoolean(item.disabled) || unchecked.length > 0, fnUuid: typeof config.fn_uuid === 'string' ? config.fn_uuid.toLowerCase() : null, managed: item.managed === true, + deployment: deploymentOf(item), unchecked }; } +// True when the local rule (from a manifest with that namespace) replaces the pulled rule. +const replaces = (local, rule) => rule.deployment !== null && rule.deployment !== undefined && + rule.deployment.namespace === local.namespace && rule.deployment.id === local.id; + /** - * The rules that run: the pulled list in its order without the rules that a - * template deploy owns (managed), and each local rule (in weight order) + * The rules that run, as a rules deploy of the local file leaves them: the + * pulled list in its order without the managed rules that a local rule + * replaces (same namespace and id), then each local rule (in weight order) * before the first pulled rule with a strictly higher weight. A local rule - * with a weight higher than all goes last. + * with a weight higher than all goes last. Every other pulled rule stays, + * also a managed rule that the file no longer lists or a managed rule of + * another namespace: a deploy does not remove them. */ -export function mergeRules(pulled, localManaged) { - const merged = pulled.filter((rule) => !rule.managed); +export function mergeRules(pulled, localRules) { + const merged = pulled.filter((rule) => !localRules.some((local) => replaces(local, rule))); const fromPortal = new Set(merged); - for (const rule of orderRules(localManaged)) { + for (const rule of orderRules(localRules)) { const at = merged.findIndex((other) => fromPortal.has(other) && other.weight > rule.weight); merged.splice(at === -1 ? merged.length : at, 0, rule); } diff --git a/src/serve/project.js b/src/serve/project.js index ffa33b6..4c65416 100644 --- a/src/serve/project.js +++ b/src/serve/project.js @@ -58,7 +58,8 @@ function readRules(root, source, functions) { const errors = ruleErrors(manifest); if (errors.length > 0) throw new Error(`${source} is not a valid rules manifest:\n ${errors.join('\n ')}`); try { - return orderRules(resolveRules(manifest, functions).rules.map(toRuntimeRule)); + const resolved = resolveRules(manifest, functions); + return orderRules(resolved.rules.map((rule) => ({ ...toRuntimeRule(rule), namespace: resolved.namespace }))); } catch (err) { throw new Error(`${source}: ${err.message}`); } @@ -93,8 +94,8 @@ function ruleWarning(rule, fn, source) { * Load and check the project files. A missing file counts as empty. * @param {Object} options * @param {string} options.root The project directory. File paths resolve from it. - * @param {Object[]|null} [options.pulledRules] --pull-rules: the portal rules in edge order. The - * rules of the rules manifest replace the managed ones (see mergeRules). Kept in memory only. + * @param {Object[]|null} [options.pulledRules] --pull-rules: the portal rules in edge order. A + * rule of the rules manifest replaces the managed rule of the same namespace and id (see mergeRules). Kept in memory only. * @param {Record|null} [options.pulledHeaders] --pull-rules: the portal global * headers. The globalHeaders of quant/serve.json override them. * @returns {{ functions: Map, rules: Object[], stores: Object[], fixtures: string, inlineFunctions: boolean, warnings: string[] }} diff --git a/tests/unit/serve/pull-rules.test.mjs b/tests/unit/serve/pull-rules.test.mjs index d3913fe..26b8a17 100644 --- a/tests/unit/serve/pull-rules.test.mjs +++ b/tests/unit/serve/pull-rules.test.mjs @@ -39,6 +39,10 @@ function item(position, action, actionConfig = {}, over = {}) { // A runtime rule with an id and weight, for the merge tests. const runtime = (id, weight, managed = false) => ({ id, weight, managed, url: ['/'], domain: ['any'], action: 'redirect', config: {} }); +// A pulled rule that a rules deploy owns: the namespace and the id of its manifest rule. +const deployed = (id, weight, namespace, ruleId) => ({ ...runtime(id, weight, true), deployment: { namespace, id: ruleId } }); +// A rule of quant/edge-rules.json with the namespace of the file. +const local = (id, weight, namespace = 'site') => ({ ...runtime(id, weight), namespace }); async function failure(promise) { try { @@ -199,6 +203,14 @@ describe('serve --pull-rules', () => { it('marks a rule that a template deploy owns', () => { expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: true })).managed).to.equal(true); }); + + it('keeps the deploy namespace and id of a managed rule only', () => { + const marker = { namespace: 'site', id: 'api', hash: 'sha256:x' }; + expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: true, deployment: marker })).deployment).to.deep.equal({ namespace: 'site', id: 'api' }); + expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: false, deployment: marker })).deployment).to.equal(null); + expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: true })).deployment).to.equal(null); + expect(fromPortalRule(item(0, 'edge_fn', {}, { managed: true, deployment: { namespace: 'site', id: 7 } })).deployment).to.equal(null); + }); }); describe('mergeRules', () => { @@ -227,11 +239,32 @@ describe('serve --pull-rules', () => { expect(ids(mergeRules(unsorted, [runtime('M', 50), runtime('N', 50)]))).to.deep.equal(['M', 'N', 'A', 'B']); }); - it('replaces the pulled managed rules with the local rules', () => { - const withManaged = [runtime('A', 0), runtime('old', 150, true), runtime('C', 200)]; - expect(ids(mergeRules(withManaged, [runtime('M', 150)]))).to.deep.equal(['A', 'M', 'C']); - expect(ids(mergeRules(withManaged, []))).to.deep.equal(['A', 'C']); + it('keeps every pulled managed rule when there is no local rule', () => { + const withManaged = [runtime('A', 0), deployed('old', 150, 'site', 'api'), runtime('C', 200), deployed('late', 50, 'site', 'web')]; + expect(ids(mergeRules(withManaged, []))).to.deep.equal(['A', 'old', 'C', 'late']); + }); + + it('keeps a managed rule that the local file no longer lists', () => { + const withManaged = [runtime('A', 0), deployed('gone', 150, 'site', 'gone'), runtime('C', 200)]; + expect(ids(mergeRules(withManaged, [local('api', 100)]))).to.deep.equal(['A', 'api', 'gone', 'C']); + }); + + it('keeps the managed rules of another namespace', () => { + const withManaged = [runtime('A', 0), deployed('other', 150, 'shop', 'api'), runtime('C', 200)]; + expect(ids(mergeRules(withManaged, [local('api', 150)]))).to.deep.equal(['A', 'other', 'api', 'C']); }); + + it('replaces the pulled copy of a local rule at its local weight', () => { + const withManaged = [runtime('A', 0), deployed('old-api', 150, 'site', 'api'), runtime('C', 200)]; + expect(ids(mergeRules(withManaged, [local('api', 250)]))).to.deep.equal(['A', 'C', 'api']); + expect(ids(mergeRules(withManaged, [local('api', 150)]))).to.deep.equal(['A', 'api', 'C']); + }); + + it('keeps a managed rule with no deployment marker', () => { + const withManaged = [runtime('A', 0), runtime('old', 150, true)]; + expect(ids(mergeRules(withManaged, [local('old', 150)]))).to.deep.equal(['A', 'old', 'old']); + }); + }); describe('pullGlobalHeaders', () => { @@ -270,6 +303,39 @@ describe('serve --pull-rules', () => { }); }); + describe('local rules over a pull', () => { + const FN_UUID = 'bbbb0000-0000-4000-8000-0000000000bb'; + let tmp; + beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-pull-local-')); }); + afterEach(() => fs.rmSync(tmp, { recursive: true, force: true })); + + const pulled = () => [ + fromPortalRule(item(0, 'redirect', {}, { uuid: 'a', weight: 0 })), + fromPortalRule(item(1, 'edge_fn', { fn_uuid: FN_UUID }, { uuid: 'm1', weight: 100, managed: true, deployment: { namespace: 'site', id: 'api', hash: 'h' } })), + fromPortalRule(item(2, 'edge_fn', { fn_uuid: FN_UUID }, { uuid: 'm2', weight: 150, managed: true, deployment: { namespace: 'site', id: 'gone', hash: 'h' } })), + fromPortalRule(item(3, 'redirect', {}, { uuid: 'c', weight: 200 })) + ]; + + it('runs the pulled list unchanged when there is no local rules file', () => { + const project = loadProject({ root: tmp, pulledRules: pulled() }); + expect(project.rules.map((rule) => rule.id)).to.deep.equal(['a', 'm1', 'm2', 'c']); + expect(project.localRules).to.equal(0); + }); + + it('replaces only the pulled rule of the same namespace and id', () => { + fs.mkdirSync(path.join(tmp, 'quant', 'edge-functions'), { recursive: true }); + fs.writeFileSync(path.join(tmp, 'quant', 'edge-functions', 'api.js'), 'return new Response(\'api\');'); + fs.writeFileSync(path.join(tmp, 'quant', 'edge-functions.json'), JSON.stringify([{ id: 'api', uuid: FN_UUID, type: 'function', path: 'quant/edge-functions/api.js', description: 'api' }])); + fs.writeFileSync(path.join(tmp, 'quant', 'edge-rules.json'), JSON.stringify({ version: 1, namespace: 'site', rules: [ + { id: 'api', type: 'function', urls: ['/api/*'], weight: 175, function_uuid: FN_UUID } + ] })); + const project = loadProject({ root: tmp, pulledRules: pulled() }); + expect(project.rules.map((rule) => rule.id)).to.deep.equal(['a', 'm2', 'api', 'c']); + expect(project.rules[2].namespace).to.equal('site'); + expect(project.localRules).to.equal(1); + }); + }); + describe('global headers from a pull', () => { let tmp; beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'quant-pull-headers-')); }); @@ -327,7 +393,7 @@ describe('serve --pull-rules', () => { const LIST = [ item(0, 'redirect', { to: '/moved', status_code: '302' }, { uuid: 'a', name: 'A', url: ['/a'], weight: 0 }), item(1, 'auth', { auth_user: 'ada', auth_pass: PASSWORD }, { uuid: 'b', name: 'B', url: ['/staff'], weight: 100 }), - item(2, 'edge_fn', { fn_uuid: FN }, { uuid: 'old', name: 'Old', url: ['/m'], weight: 150, managed: true }), + item(2, 'edge_fn', { fn_uuid: FN }, { uuid: 'old', name: 'Old', url: ['/m'], weight: 150, managed: true, deployment: { namespace: 'test', id: 'local-m', hash: 'h' } }), item(4, 'proxy', { to: 'http://127.0.0.1:1$*', inject_headers: { 'x-shared': SECRET }, disable_ssl_verify: 'true' }, { uuid: 'p', name: 'P', url: ['/api/*'], weight: 160 }), item(5, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'pulled C' }, { uuid: 'c', name: 'C', url: ['/m', '/n'], weight: 200 }), item(6, 'custom_response', { custom_response_status_code: '200', custom_response_body: 'with cookie' }, { uuid: 'k', name: 'K', url: ['/cookie'], weight: 300, only_with_cookie: 'preview' }) From b92a2e7deb4fc8d124b63957be7c1d2d1762f353 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 23:16:29 +1000 Subject: [PATCH 27/32] fix(serve): share the portal request budget and soften header read errors - One request budget of 200 for the whole start-up, shared by the rules, headers and KV reads. The client checks it before each attempt, 429 retries included, and refuses the attempt once it is spent. The KV read then stops as truncated (request limit). - A 5xx or no answer from the project headers gives one warning and no portal global headers instead of a stop. The 403 warning names the headers read. - static_error_page is on for any value that is not empty. - readStore no longer takes a request count or returns one. --- README.md | 4 +- src/commands/serve.js | 12 ++-- src/serve/kv/remote.js | 16 ++--- src/serve/portal/client.js | 28 +++++++- src/serve/portal/kv-remote.js | 89 +++++++++++------------- src/serve/portal/pull-rules.js | 31 ++++++--- tests/unit/serve/kv-remote-read.test.mjs | 84 ++++++++++++++++------ tests/unit/serve/portal-client.test.mjs | 48 +++++++++---- tests/unit/serve/pull-rules.test.mjs | 36 ++++++++-- 9 files changed, 235 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index be67556..080c841 100644 --- a/README.md +++ b/README.md @@ -481,7 +481,7 @@ quant serve dist --pull-rules --kv-remote sessions --org acme --project marketin - `QUANT_API_TOKEN` and `QUANT_BASE_URL` are the only source of the token and the portal URL. There is no flag for either, and `quant serve` never prints the token. - `QUANT_BASE_URL` must be an HTTPS portal `/api/v2` URL. Plain HTTP is allowed only for `localhost`, `127.0.0.1` and `[::1]`. - The organisation and project come from `--org` and `--project`, then `quant/serve.json`, then `quant.json`. If none sets them, the command stops with a message that names the flag. -- Scopes: `rules:read` for the rules, `projects:read` for the global headers, and `kv:read` for the KV store. Without `rules:read` or `kv:read`, `quant serve` stops and the message names the scope. Without `projects:read`, `quant serve` prints one warning and uses only the local global headers. +- Scopes: `rules:read` for the rules, `projects:read` for the global headers, and `kv:read` for the KV store. Without `rules:read` or `kv:read`, `quant serve` stops and the message names the scope. Without `projects:read`, `quant serve` prints one warning and uses only the local global headers. If the portal cannot give the global headers (a 5xx answer or no answer), `quant serve` also prints one warning and uses only the local global headers. ### What the snapshot is @@ -516,7 +516,7 @@ Other points: `--kv-remote ` copies the KV store `` into the local KV, in memory. The store must be linked to the portal project and listed in `quant/meta.json`, so that functions can open it. The `initial_values` and fixtures of that store are not seeded. -- Caps: at most 10,000 keys and 200 portal requests. If a cap stops the read, `quant serve` prints `KV store : copied the first keys only (limit 10000). Writes stay local.`. For the request cap, the text in the brackets is `request limit 200`. If the portal ends the key list early, it is `the portal ended the list early`. +- Caps: at most 10,000 keys and 200 portal requests. The 200 requests are for the full start-up: the `--pull-rules` requests and each 429 retry count too. If a cap stops the read, `quant serve` prints `KV store : copied the first keys only (limit 10000). Writes stay local.`. For the request cap, the text in the brackets is `request limit 200`. If the portal ends the key list early, it is `the portal ended the list early`. - The start-up output shows `KV store : copied keys from the portal ( encrypted values masked).` If the portal store has ledger entries, the line adds ` ( ledger entries not copied)` before the final full stop. - A secret value reads `[ENCRYPTED]`. The portal does not send the real value. - Index definitions are copied, so index lists and counts work. Expiry times are kept. diff --git a/src/commands/serve.js b/src/commands/serve.js index 4e6ae48..c990575 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -15,7 +15,7 @@ import { DEFAULT_FILES, loadProject } from '../serve/project.js'; import { openLocalKv } from '../serve/kv/wire.js'; import { LEDGERS_FILE, readLedgerFile } from '../serve/kv/ledgers.js'; import { readServeConfig } from '../serve/project-config.js'; -import { createPortalClient, portalOptions } from '../serve/portal/client.js'; +import { createPortalClient, portalOptions, requestBudget } from '../serve/portal/client.js'; import { pullProject } from '../serve/portal/pull-rules.js'; import { readRemoteStore, remoteLines } from '../serve/kv/remote.js'; @@ -475,10 +475,10 @@ async function localKv(options, { log, print, portal }) { // --pull-rules: read the portal rules and global headers one time. They // stay in memory only. QUANT_API_TOKEN and QUANT_BASE_URL come from the // environment; the org and project are the serve identity. -async function pulledProject(options, { env = process.env, fetchImpl } = {}) { +async function pulledProject(options, { env = process.env, fetchImpl, budget } = {}) { if (!options.pullRules) return null; const { customer, project, token, base } = portalOptions({ clientid: options.org, project: options.project }, env, '--pull-rules'); - const client = createPortalClient({ base, token, ...(fetchImpl ? { fetchImpl } : {}) }); + const client = createPortalClient({ base, token, budget, ...(fetchImpl ? { fetchImpl } : {}) }); return pullProject(client, { customer, project }); } @@ -564,8 +564,10 @@ function watchRun(server, child) { */ export async function runServe(options, { stop, print = console.log, log, kvStore, onListening, env, fetchImpl } = {}) { if (options.build) runBuild(options.build, options.root); - const pulled = await pulledProject(options, { env, fetchImpl }); - const kv = await kvFor(options, { kvStore, log, print, portal: { env, fetchImpl } }); + // One request budget for every portal read of the start-up. + const budget = requestBudget(); + const pulled = await pulledProject(options, { env, fetchImpl, budget }); + const kv = await kvFor(options, { kvStore, log, print, portal: { env, fetchImpl, budget } }); const server = await startOrShowKeys(serverOptions(withPull(options, pulled), { kvStore: kv.kvStore, backend: kv.backend, log }, print), kv, print); const pullWarnings = pulled ? pulled.warnings : []; const summary = { ...server.summary, warnings: [...(options.configWarnings || []), ...pullWarnings, ...server.summary.warnings] }; diff --git a/src/serve/kv/remote.js b/src/serve/kv/remote.js index ed60577..50665d2 100644 --- a/src/serve/kv/remote.js +++ b/src/serve/kv/remote.js @@ -6,11 +6,11 @@ */ import { createLocalKv } from '@quantcdn/edge-kv'; -import { createPortalClient, portalOptions } from '../portal/client.js'; +import { createPortalClient, portalOptions, requestBudget, START_REQUEST_LIMIT } from '../portal/client.js'; import { findStore, readStore } from '../portal/kv-remote.js'; export const REMOTE_MAX_KEYS = 10000; -export const REMOTE_MAX_REQUESTS = 200; +export const REMOTE_MAX_REQUESTS = START_REQUEST_LIMIT; const MASKED = '[ENCRYPTED]'; // Portal index states that do not serve queries: they are not declared. const UNUSED_STATES = ['FAILED', 'DROPPING']; @@ -35,18 +35,16 @@ export function toLocalIndex(def) { /** * Read the store from the portal: the store list, the index definitions, - * the keys and their values. The request count includes 429 retries. + * the keys and their values. `budget` is the request budget of the start-up, + * shared with --pull-rules; each attempt (429 retries too) uses one request. * A PortalError stops the start-up with its one-line message. */ -export async function readRemoteStore({ name, org, project }, { env = process.env, fetchImpl } = {}) { +export async function readRemoteStore({ name, org, project }, { env = process.env, fetchImpl, budget = requestBudget() } = {}) { const ids = portalOptions({ clientid: org, project }, env, '--kv-remote'); - const counter = { count: 0 }; - const client = createPortalClient({ - base: ids.base, token: ids.token, onRequest: () => { counter.count += 1; }, ...(fetchImpl ? { fetchImpl } : {}) - }); + const client = createPortalClient({ base: ids.base, token: ids.token, budget, ...(fetchImpl ? { fetchImpl } : {}) }); const ctx = { customer: ids.customer, project: ids.project, name }; const store = await findStore(client, ctx); - const snapshot = await readStore(client, { ...ctx, counter }, store.id, { maxKeys: REMOTE_MAX_KEYS, maxRequests: REMOTE_MAX_REQUESTS }); + const snapshot = await readStore(client, ctx, store.id, { maxKeys: REMOTE_MAX_KEYS }); return { name, format: store.format ?? 2, ...snapshot }; } diff --git a/src/serve/portal/client.js b/src/serve/portal/client.js index 719edba..62133df 100644 --- a/src/serve/portal/client.js +++ b/src/serve/portal/client.js @@ -6,6 +6,8 @@ const MAX_WAIT_MS = 30000; const MIN_RETRY_MS = 250; const MAX_MESSAGE = 300; const LOCAL_HOSTS = ['localhost', '127.0.0.1', '[::1]']; +// The most portal requests of one start-up (rules, headers and KV together). +export const START_REQUEST_LIMIT = 200; export class PortalError extends Error { constructor(message, status) { @@ -15,6 +17,25 @@ export class PortalError extends Error { } } +/** The portal refused no request: quant serve spent its request budget first. */ +export class RequestLimitError extends PortalError { + constructor(max) { + super(`quant serve reached its limit of ${max} portal requests at start.`); + this.name = 'RequestLimitError'; + } +} + +/** A request budget that clients can share: each HTTP attempt uses one. */ +export function requestBudget(max = START_REQUEST_LIMIT) { + return { max, used: 0 }; +} + +// Use one request of the budget, or refuse the attempt when it is spent. +function spend(budget) { + if (budget.used >= budget.max) throw new RequestLimitError(budget.max); + budget.used += 1; +} + // A string as one short line, or null when it is not text. export function oneLine(value) { if (typeof value !== 'string' || value.trim() === '') return null; @@ -121,13 +142,14 @@ async function settle(context, response, scopeInfo) { return { status, body: body ?? null }; } -// onRequest() is called once for each HTTP attempt, a 429 retry included. -export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), onRequest = () => {} }) { +// Each HTTP attempt, a 429 retry included, uses one request of `budget`. +// Clients of one start-up share one budget. +export function createPortalClient({ base, token, fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), budget = requestBudget() }) { const context = { base, token, fetchImpl }; async function request(call, scopeInfo) { let waited = 0; for (;;) { - onRequest(); + spend(budget); const response = await send(context, call); if (response.status !== 429) return settle(context, response, scopeInfo); const delay = retryDelayMs(response); diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js index 8a61c35..e31d8c8 100644 --- a/src/serve/portal/kv-remote.js +++ b/src/serve/portal/kv-remote.js @@ -1,4 +1,4 @@ -import { PortalError } from './client.js'; +import { PortalError, RequestLimitError } from './client.js'; const PAGE_LIMIT = 1000; const BATCH_SIZE = 100; @@ -19,20 +19,7 @@ export async function findStore(client, { customer, project, name }) { return { id: found.id, name: found.name, ...([1, 2].includes(found.format) ? { format: found.format } : {}) }; } -// Counts every request. With a counter (the onRequest hook of the client: -// { count }), the count is the counter, so 429 retries count too; else each -// call counts one, from `used`. `full` is true once the request cap is reached. -function budget(max, used, counter) { - const own = { count: used }; - const source = counter || own; - const state = { max, tick: () => { own.count += 1; } }; - Object.defineProperty(state, 'requests', { get: () => source.count }); - state.full = () => state.requests >= state.max; - return state; -} - -async function fetchOk(state, name, send) { - state.tick(); +async function fetchOk(name, send) { const { status, body } = await send(scopeInfo(name)); if (status === 404) throw notLinked(name); return body; @@ -43,66 +30,72 @@ function splitItem(item) { return { key, value, metadata }; } -async function readValues(client, state, ctx, id, keys) { - const entries = []; +// Reads the values of `keys` into `entries`, 100 keys for each request. +async function readValues(client, ctx, id, keys, entries) { for (let start = 0; start < keys.length; start += BATCH_SIZE) { - if (state.full()) return { entries, complete: false }; const chunk = keys.slice(start, start + BATCH_SIZE); - const body = await fetchOk(state, ctx.name, info => client.post(storePath(ctx, `/${encodeURIComponent(id)}/batch-get`), { keys: chunk }, info)); + const body = await fetchOk(ctx.name, info => client.post(storePath(ctx, `/${encodeURIComponent(id)}/batch-get`), { keys: chunk }, info)); const items = Array.isArray(body?.items) ? body.items : []; for (const item of items) if (item && typeof item.key === 'string') entries.push(splitItem(item)); } - return { entries, complete: true }; } -function readIndexes(client, state, ctx, id) { - return fetchOk(state, ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/indexes`), undefined, info)) +function readIndexes(client, ctx, id) { + return fetchOk(ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/indexes`), undefined, info)) .then(body => (Array.isArray(body?.indexes) ? body.indexes : [])); } -function readKeysPage(client, state, ctx, id, cursor) { +function readKeysPage(client, ctx, id, cursor) { const query = { limit: PAGE_LIMIT, ...(cursor ? { cursor } : {}) }; - return fetchOk(state, ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/keys`), query, info)); + return fetchOk(ctx.name, info => client.get(storePath(ctx, `/${encodeURIComponent(id)}/keys`), query, info)); } const pageKeys = page => (Array.isArray(page?.data) ? page.data : []).map(row => row?.key).filter(key => typeof key === 'string'); -// Why a page stops the read: 'requests' (the request cap cut the values), -// 'keys' (maxKeys), 'cursor' (has_more with no cursor: the rest cannot be -// read), or null (the read goes on, or the list has ended). -function stopReason({ complete, room, count, hasMore, more }) { - if (!complete) return 'requests'; +// Why a page stops the read: 'keys' (maxKeys), 'cursor' (has_more with no +// cursor: the rest cannot be read), or null (the read goes on, or the list +// has ended). The request budget stops the read with 'requests' (see readStore). +function stopReason({ room, count, hasMore, more }) { if (room < count || (more && room <= count)) return 'keys'; return hasMore && !more ? 'cursor' : null; } -// Reads one page of keys and their values. Returns the next cursor, or the stop reason. -async function readPage(client, state, ctx, id, { cursor, room }) { - const page = await readKeysPage(client, state, ctx, id, cursor); +// Reads one page of keys and their values into `entries`. Returns the next cursor, or the stop reason. +async function readPage(client, ctx, id, { cursor, room, entries }) { + const page = await readKeysPage(client, ctx, id, cursor); const all = pageKeys(page); - const values = await readValues(client, state, ctx, id, all.slice(0, room)); + await readValues(client, ctx, id, all.slice(0, room), entries); const hasMore = page?.has_more === true; const more = hasMore && Boolean(page.next_cursor); - const stop = stopReason({ complete: values.complete, room, count: all.length, hasMore, more }); - return { entries: values.entries, next: more && !stop ? page.next_cursor : null, stop }; + const stop = stopReason({ room, count: all.length, hasMore, more }); + return { next: more && !stop ? page.next_cursor : null, stop }; } -export async function readStore(client, ctx, id, { maxKeys = 10000, maxRequests = 200 } = {}) { - const state = budget(maxRequests, ctx.requestsUsed ?? 0, ctx.counter); - const entries = []; - const indexes = state.full() ? [] : await readIndexes(client, state, ctx, id); +async function readPages(client, ctx, id, { maxKeys, entries }) { let cursor; - let stoppedBy = null; + let page; do { - if (state.full()) { - stoppedBy = 'requests'; - break; - } - const page = await readPage(client, state, ctx, id, { cursor, room: maxKeys - entries.length }); - entries.push(...page.entries); - stoppedBy = page.stop; + page = await readPage(client, ctx, id, { cursor, room: maxKeys - entries.length, entries }); cursor = page.next; } while (cursor); + return page.stop; +} + +/** + * Read the index definitions, the keys and the values of a store. The + * request budget of the client caps the read: when it is spent, the read + * stops with the entries so far (truncated, stoppedBy 'requests'). + */ +export async function readStore(client, ctx, id, { maxKeys = 10000 } = {}) { + const entries = []; + const read = { indexes: [], stoppedBy: null }; + try { + read.indexes = await readIndexes(client, ctx, id); + read.stoppedBy = await readPages(client, ctx, id, { maxKeys, entries }); + } catch (err) { + if (!(err instanceof RequestLimitError)) throw err; + read.stoppedBy = 'requests'; + } const masked = entries.filter(entry => entry.value === MASKED).length; - return { entries, indexes, masked, truncated: stoppedBy !== null, stoppedBy, requests: state.requests }; + return { entries, indexes: read.indexes, masked, truncated: read.stoppedBy !== null, stoppedBy: read.stoppedBy }; } diff --git a/src/serve/portal/pull-rules.js b/src/serve/portal/pull-rules.js index 542074c..9145ed8 100644 --- a/src/serve/portal/pull-rules.js +++ b/src/serve/portal/pull-rules.js @@ -7,15 +7,16 @@ * settings can hold credentials, so no message prints a setting value. */ -import { PortalError } from './client.js'; +import { PortalError, RequestLimitError } from './client.js'; import { orderRules } from '../rules.js'; export const PARTIAL_WARNING = 'Some rule types are not in the portal response for this organisation, so they do not run locally.'; -export const HEADERS_SCOPE_WARNING = 'QUANT_API_TOKEN lacks projects:read, so the portal\'s global headers are not used.'; +export const HEADERS_SCOPE_WARNING = 'QUANT_API_TOKEN cannot read project headers (projects:read), so the portal\'s global headers are not used.'; +export const HEADERS_READ_WARNING = 'The portal\'s global headers could not be read, so they are not used.'; // Action settings that are booleans, but can arrive as text. const BOOLEAN_SETTINGS = [ - 'only_proxy_404', 'static_error_page', 'waf_enabled', 'disable_ssl_verify', 'failover_mode', + 'only_proxy_404', 'waf_enabled', 'disable_ssl_verify', 'failover_mode', 'proxy_alert_enabled', 'proxy_inline_fn_enabled', 'application_proxy' ]; // Action settings that are lists of status codes. @@ -65,6 +66,8 @@ function normalConfig(value) { const config = isObject(value) ? { ...value } : {}; for (const name of BOOLEAN_SETTINGS.filter((key) => key in config)) config[name] = toBoolean(config[name]); for (const name of STATUS_LISTS.filter((key) => key in config)) config[name] = toList(config[name]); + // Static error pages are on for any value that is not empty. + if ('static_error_page' in config) config.static_error_page = ![undefined, null, false, ''].includes(config.static_error_page); return config; } @@ -175,24 +178,32 @@ export function mergeRules(pulled, localRules) { return merged; } -function headersBody(body) { +function headersBody(body, status) { if (Array.isArray(body) && body.length === 0) return {}; - if (!isObject(body)) throw new PortalError('The portal sent project headers that are not valid.'); + if (!isObject(body)) throw new PortalError('The portal sent project headers that are not valid.', status); return body; } +// The warning when the headers cannot be read, or null when the error stops the start-up. +function headersWarning(err) { + if (!(err instanceof PortalError) || err instanceof RequestLimitError) return null; + if (err.status === 403) return HEADERS_SCOPE_WARNING; + return err.status === undefined || err.status >= 500 ? HEADERS_READ_WARNING : null; +} + /** - * Read the global headers of the project. A token without projects:read - * gives a warning and null. + * Read the global headers of the project. A token without projects:read, + * a portal error (5xx) or no answer gives one warning and null. * @returns {Promise|null>} */ export async function pullGlobalHeaders(client, { customer, project, warn = () => {} }) { try { const { status, body } = await client.get(projectPath({ customer, project }, 'custom-headers'), undefined, { scope: 'projects:read', what: 'project headers' }); - return status === 404 ? null : headersBody(body); + return status === 404 ? null : headersBody(body, status); } catch (err) { - if (!(err instanceof PortalError) || err.status !== 403) throw err; - warn(HEADERS_SCOPE_WARNING); + const warning = headersWarning(err); + if (warning === null) throw err; + warn(warning); return null; } } diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs index 5633c6a..b3106ec 100644 --- a/tests/unit/serve/kv-remote-read.test.mjs +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -1,6 +1,7 @@ import { expect } from 'chai'; import { findStore, readStore } from '../../../src/serve/portal/kv-remote.js'; -import { createPortalClient } from '../../../src/serve/portal/client.js'; +import { createPortalClient, requestBudget } from '../../../src/serve/portal/client.js'; +import { pullProject } from '../../../src/serve/portal/pull-rules.js'; const BASE = new URL('http://127.0.0.1:8080/api/v2'); const CTX = { customer: 'org', project: 'proj', name: 'sessions' }; @@ -16,7 +17,7 @@ const reply = (status, body) => ({ }); // A fake portal: pages of keys, batch-get, indexes and the store list. -function portal({ total = 0, pageSize = 1000, stores, indexes = [], status = {}, encrypted = [] } = {}) { +function portal({ total = 0, pageSize = 1000, stores, indexes = [], status = {}, encrypted = [], budget } = {}) { const calls = []; const keys = Array.from({ length: total }, (_, i) => `k${String(i).padStart(5, '0')}`); const fetchImpl = async (url, init) => { @@ -38,7 +39,7 @@ function portal({ total = 0, pageSize = 1000, stores, indexes = [], status = {}, } return reply(404); }; - return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {} }) }; + return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {}, ...(budget ? { budget } : {}) }) }; } const rejection = async (promise) => { @@ -97,8 +98,8 @@ describe('remote KV read', () => { expect(batches.map(c => c.body.keys.length)).to.deep.equal([100, 100, 50]); expect(batches.every(c => c.method === 'POST')).to.equal(true); expect(result.entries[0]).to.deep.equal({ key: 'k00000', value: { n: 'k00000' }, metadata: { encrypted: false, version: 'V1', updated_at: 't' } }); - expect(result.requests).to.equal(calls.length); - expect(result.requests).to.equal(1 + 3 + 1); + expect(calls).to.have.length(1 + 1 + 3); + expect(result).to.not.have.property('requests'); }); it('reads index definitions and counts them as a request', async () => { @@ -135,15 +136,43 @@ describe('remote KV read', () => { expect(result.stoppedBy).to.equal('keys'); }); - it('stops at maxRequests and sets truncated', async () => { - const { client, calls } = portal({ total: 5000 }); - const result = await readStore(client, CTX, 'abc', { maxRequests: 4 }); + it('stops at the request budget of the client and sets truncated', async () => { + const { client, calls } = portal({ total: 5000, budget: requestBudget(4) }); + const result = await readStore(client, CTX, 'abc'); expect(calls).to.have.length(4); - expect(result.requests).to.equal(4); + expect(result.entries).to.have.length(200); expect(result.truncated).to.equal(true); expect(result.stoppedBy).to.equal('requests'); }); + it('keeps the indexes and no keys when the budget is spent before the keys', async () => { + const indexes = [{ name: 'byX', collection: 'c', fields: ['x'], state: 'ACTIVE' }]; + const { client, calls } = portal({ total: 5, indexes, budget: requestBudget(1) }); + const result = await readStore(client, CTX, 'abc'); + expect(calls).to.have.length(1); + expect(result.indexes).to.deep.equal(indexes); + expect(result.entries).to.deep.equal([]); + expect(result.stoppedBy).to.equal('requests'); + const spent = await readStore(portal({ total: 5, budget: { max: 1, used: 1 } }).client, CTX, 'abc'); + expect(spent).to.include({ truncated: true, stoppedBy: 'requests' }); + expect(spent.indexes).to.deep.equal([]); + }); + + it('counts the rules and headers requests toward the shared start-up budget', async () => { + const budget = requestBudget(5); + const rulesFetch = async (url) => (String(url).endsWith('/rules') ? reply(200, []) : reply(200, {})); + await pullProject(createPortalClient({ base: BASE, token: 'tok', fetchImpl: rulesFetch, budget }), { customer: 'org', project: 'proj' }); + expect(budget.used).to.equal(2); + const { client, calls } = portal({ total: 300, budget }); + await findStore(client, CTX); + const result = await readStore(client, CTX, 'abc'); + // 2 rules and headers + 1 store list + 1 indexes + 1 keys page: no batch is left. + expect(calls).to.have.length(3); + expect(budget.used).to.equal(5); + expect(result.entries).to.deep.equal([]); + expect(result.stoppedBy).to.equal('requests'); + }); + it('reports not linked on 404 from a store route', async () => { const { client } = portal({ status: { '/abc/keys': 404 } }); const error = await rejection(readStore(client, CTX, 'abc')); @@ -174,7 +203,7 @@ describe('remote KV read', () => { }); // A fake portal with one fixed keys page: `page` is the keys reply. - function onePage(page) { + function onePage(page, budget) { const calls = []; const fetchImpl = async (url, init) => { const route = new URL(String(url)).pathname.replace(PREFIX, ''); @@ -183,7 +212,7 @@ describe('remote KV read', () => { if (route === '/abc/batch-get') return reply(200, { items: JSON.parse(init.body).keys.map(key => ({ key, value: 1 })) }); return reply(200, { indexes: [] }); }; - return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl }) }; + return { calls, client: createPortalClient({ base: BASE, token: 'tok', fetchImpl, ...(budget ? { budget } : {}) }) }; } it('stops with truncated when has_more is true and next_cursor is missing or empty', async () => { @@ -198,18 +227,19 @@ describe('remote KV read', () => { }); it('stops a repeated cursor at the request cap', async () => { - const { client, calls } = onePage({ data: [{ key: 'a' }], has_more: true, next_cursor: 'same' }); - const result = await readStore(client, CTX, 'abc', { maxRequests: 9 }); + const { client, calls } = onePage({ data: [{ key: 'a' }], has_more: true, next_cursor: 'same' }, requestBudget(9)); + const result = await readStore(client, CTX, 'abc'); expect(calls).to.have.length(9); - expect(result.requests).to.equal(9); expect(result.truncated).to.equal(true); expect(result.stoppedBy).to.equal('requests'); }); - it('counts 429 retries in requests and in the cap through onRequest', async () => { - const counter = { count: 0 }; + it('counts 429 retries in the request budget', async () => { + const budget = requestBudget(6); + let fetched = 0; let limited = 0; const fetchImpl = async (url, init) => { + fetched += 1; const route = new URL(String(url)).pathname.replace(PREFIX, '') || '/'; if (route === '/') return reply(200, [{ id: 'abc', name: 'sessions' }]); if (route === '/abc/indexes' && limited++ < 2) return { status: 429, headers: { get: () => '0' }, json: async () => ({}) }; @@ -217,13 +247,27 @@ describe('remote KV read', () => { if (route === '/abc/keys') return reply(200, { data: [{ key: 'a' }], has_more: true, next_cursor: 'n' }); return reply(200, { items: JSON.parse(init.body).keys.map(key => ({ key, value: 1 })) }); }; - const client = createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {}, onRequest: () => { counter.count += 1; } }); + const client = createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {}, budget }); await findStore(client, CTX); - const result = await readStore(client, { ...CTX, counter }, 'abc', { maxRequests: 6 }); + const result = await readStore(client, CTX, 'abc'); // 1 store list + 3 index attempts + 1 keys page + 1 batch: the cap of 6 stops the read. - expect(counter.count).to.equal(6); - expect(result.requests).to.equal(6); + expect(fetched).to.equal(6); + expect(budget.used).to.equal(6); + expect(result.entries).to.have.length(1); expect(result.truncated).to.equal(true); + expect(result.stoppedBy).to.equal('requests'); + }); + + it('stops 429 retries inside one call at the cap', async () => { + let fetched = 0; + const fetchImpl = async () => { + fetched += 1; + return { status: 429, headers: { get: () => '0' }, json: async () => ({}) }; + }; + const client = createPortalClient({ base: BASE, token: 'tok', fetchImpl, sleep: async () => {}, budget: requestBudget(4) }); + const result = await readStore(client, CTX, 'abc'); + expect(fetched).to.equal(4); + expect(result).to.include({ truncated: true, stoppedBy: 'requests' }); }); it('names kv:read on 403 and never prints a value in errors', async () => { diff --git a/tests/unit/serve/portal-client.test.mjs b/tests/unit/serve/portal-client.test.mjs index d541056..0ba81c2 100644 --- a/tests/unit/serve/portal-client.test.mjs +++ b/tests/unit/serve/portal-client.test.mjs @@ -1,7 +1,7 @@ import { inspect } from 'node:util'; import { expect } from 'chai'; import { rulesOptions } from '../../../src/commands/rules.js'; -import { portalOptions, createPortalClient, PortalError } from '../../../src/serve/portal/client.js'; +import { portalOptions, createPortalClient, PortalError, RequestLimitError, requestBudget, START_REQUEST_LIMIT } from '../../../src/serve/portal/client.js'; const TOKEN = 'test-token'; const BASE = new URL('http://127.0.0.1:8080/api/v2'); @@ -263,33 +263,57 @@ describe('portal client', () => { }); }); - describe('onRequest', () => { - function counted(responses) { + describe('request budget', () => { + function counted(responses, budget = requestBudget()) { const queue = [...responses]; - let count = 0; + let fetched = 0; const fetchImpl = async () => { + fetched += 1; const next = queue.shift(); if (next instanceof Error) throw next; return next; }; - const client = createPortalClient({ base: BASE, token: TOKEN, fetchImpl, sleep: async () => {}, onRequest: () => { count += 1; } }); - return { client, count: () => count }; + const client = createPortalClient({ base: BASE, token: TOKEN, fetchImpl, sleep: async () => {}, budget }); + return { client, budget, fetched: () => fetched }; } - it('is called once for each HTTP attempt, retries included', async () => { + it('counts each HTTP attempt, retries included', async () => { const get = counted([reply(429, {}, { 'retry-after': '1' }), reply(429, {}, { 'retry-after': '1' }), reply(200, {})]); await get.client.get('/x'); - expect(get.count()).to.equal(3); + expect(get.budget.used).to.equal(3); const post = counted([reply(429, {}, { 'retry-after': '1' }), reply(200, {})]); await post.client.post('/x', {}); - expect(post.count()).to.equal(2); + expect(post.budget.used).to.equal(2); }); - it('counts a failed attempt and works without the hook', async () => { + it('counts a failed attempt and has a default cap of 200', async () => { const down = counted([new Error('boom')]); await failure(down.client.get('/x')); - expect(down.count()).to.equal(1); - expect(await setup([reply(200, { ok: 1 })]).client.get('/x')).to.deep.equal({ status: 200, body: { ok: 1 } }); + expect(down.budget.used).to.equal(1); + expect(requestBudget()).to.deep.equal({ max: 200, used: 0 }); + expect(START_REQUEST_LIMIT).to.equal(200); + }); + + it('stops 429 retries inside one call at the cap', async () => { + const limited = Array.from({ length: 10 }, () => reply(429, {}, { 'retry-after': '0' })); + const capped = counted(limited, requestBudget(3)); + const error = await failure(capped.client.get('/x')); + expect(capped.fetched()).to.equal(3); + expect(error).to.be.instanceOf(RequestLimitError); + expect(error).to.be.instanceOf(PortalError); + expect(error.message).to.equal('quant serve reached its limit of 3 portal requests at start.'); + }); + + it('shares one budget between clients', async () => { + const budget = requestBudget(2); + const one = counted([reply(200, {})], budget); + const two = counted([reply(200, {}), reply(200, {})], budget); + await one.client.get('/x'); + await two.client.get('/x'); + const error = await failure(two.client.get('/x')); + expect(error).to.be.instanceOf(RequestLimitError); + expect(two.fetched()).to.equal(1); + expect(budget.used).to.equal(2); }); }); }); diff --git a/tests/unit/serve/pull-rules.test.mjs b/tests/unit/serve/pull-rules.test.mjs index 26b8a17..baf2c38 100644 --- a/tests/unit/serve/pull-rules.test.mjs +++ b/tests/unit/serve/pull-rules.test.mjs @@ -5,7 +5,7 @@ import path from 'node:path'; import { createPortalClient, PortalError } from '../../../src/serve/portal/client.js'; import { fromPortalRule, mergeRules, pullGlobalHeaders, pullProject, pullRules, - HEADERS_SCOPE_WARNING, PARTIAL_WARNING + HEADERS_READ_WARNING, HEADERS_SCOPE_WARNING, PARTIAL_WARNING } from '../../../src/serve/portal/pull-rules.js'; import { loadProject } from '../../../src/serve/project.js'; import { resolveOptions, runServe } from '../../../src/commands/serve.js'; @@ -184,15 +184,22 @@ describe('serve --pull-rules', () => { it('makes booleans of boolean settings sent as text', () => { const rule = fromPortalRule(item(0, 'proxy', { - only_proxy_404: 'true', static_error_page: '', waf_enabled: '0', disable_ssl_verify: '1', failover_mode: 'false' + only_proxy_404: 'true', static_error_page: 'false', waf_enabled: '0', disable_ssl_verify: '1', failover_mode: 'false' }, { disabled: 'true' })); expect(rule.disabled).to.equal(true); expect(rule.config).to.include({ - only_proxy_404: true, static_error_page: false, waf_enabled: false, disable_ssl_verify: true, failover_mode: false + only_proxy_404: true, static_error_page: true, waf_enabled: false, disable_ssl_verify: true, failover_mode: false }); expect(fromPortalRule(item(0, 'redirect', {}, { disabled: '0' })).disabled).to.equal(false); }); + it('turns static error pages on for any value that is not empty', () => { + const staticPages = (value) => fromPortalRule(item(0, 'proxy', { static_error_page: value })).config.static_error_page; + for (const value of ['yes', 'true', '1', '0', 'off', true]) expect(staticPages(value), String(value)).to.equal(true); + for (const value of ['', false, null]) expect(staticPages(value), String(value)).to.equal(false); + expect(fromPortalRule(item(0, 'proxy', {})).config).to.not.have.property('static_error_page'); + }); + it('makes status_codes a list', () => { expect(fromPortalRule(item(0, 'error_page', { status_codes: '404' })).config.status_codes).to.deep.equal(['404']); expect(fromPortalRule(item(0, 'error_page', { status_codes: 500 })).config.status_codes).to.deep.equal([500]); @@ -279,7 +286,28 @@ describe('serve --pull-rules', () => { const { client } = fakePortal({ '/custom-headers': reply(403, {}) }); expect(await pullGlobalHeaders(client, { ...IDS, warn: (line) => warnings.push(line) })).to.equal(null); expect(warnings).to.deep.equal([HEADERS_SCOPE_WARNING]); - expect(HEADERS_SCOPE_WARNING).to.equal('QUANT_API_TOKEN lacks projects:read, so the portal\'s global headers are not used.'); + expect(HEADERS_SCOPE_WARNING).to.equal('QUANT_API_TOKEN cannot read project headers (projects:read), so the portal\'s global headers are not used.'); + }); + + it('warns and gives null when the portal fails or does not answer', async () => { + expect(HEADERS_READ_WARNING).to.equal('The portal\'s global headers could not be read, so they are not used.'); + for (const status of [500, 502, 503]) { + const warnings = []; + const { client } = fakePortal({ '/custom-headers': reply(status, { message: 'down' }) }); + expect(await pullGlobalHeaders(client, { ...IDS, warn: (line) => warnings.push(line) }), String(status)).to.equal(null); + expect(warnings).to.deep.equal([HEADERS_READ_WARNING]); + } + const warnings = []; + const down = createPortalClient({ base: BASE, token: TOKEN, fetchImpl: async () => { throw new Error('offline'); } }); + expect(await pullGlobalHeaders(down, { ...IDS, warn: (line) => warnings.push(line) })).to.equal(null); + expect(warnings).to.deep.equal([HEADERS_READ_WARNING]); + }); + + it('still stops on a refused token or a body that is not valid', async () => { + const refused = await failure(pullGlobalHeaders(fakePortal({ '/custom-headers': reply(401, {}) }).client, IDS)); + expect(refused.status).to.equal(401); + const invalid = await failure(pullGlobalHeaders(fakePortal({ '/custom-headers': reply(200, 'x') }).client, IDS)); + expect(invalid.message).to.equal('The portal sent project headers that are not valid.'); }); it('takes an empty list as no headers', async () => { From 1967fab97f5590e171e6d6b48efdec7dc051bbd6 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 23:21:03 +1000 Subject: [PATCH 28/32] refactor(serve): one copy of the shared rule helpers ruleLabel comes from rule-actions.js, listsStatus from rules.js, and isObject from src/helper/is-object.js. No behaviour change. --- src/helper/functions-manifest.js | 6 ++---- src/helper/is-object.js | 2 ++ src/helper/quant-meta.js | 3 ++- src/serve/inline.js | 2 +- src/serve/kv/ledgers.js | 2 +- src/serve/kv/remote.js | 3 ++- src/serve/portal/pull-rules.js | 5 ++--- src/serve/project-config.js | 2 +- src/serve/rule-actions.js | 4 +--- src/serve/rules.js | 9 +++------ src/serve/seed.js | 2 +- 11 files changed, 18 insertions(+), 22 deletions(-) create mode 100644 src/helper/is-object.js diff --git a/src/helper/functions-manifest.js b/src/helper/functions-manifest.js index 3d0d24c..40ab6c8 100644 --- a/src/helper/functions-manifest.js +++ b/src/helper/functions-manifest.js @@ -6,6 +6,8 @@ * its function by an explicit uuid: no tool derives one. */ +import { isObject } from './is-object.js'; + /** * UUID shape only (8-4-4-4-12 hex). The version and variant digits are not * checked: deployed manifests hold UUIDs with other version digits, and the @@ -20,10 +22,6 @@ export const FUNCTION_ID = /^[a-z][a-z0-9-]{0,47}$/; export const FUNCTION_TYPES = ['function', 'edge', 'auth', 'filter']; -function isObject(value) { - return value !== null && typeof value === 'object' && !Array.isArray(value); -} - function normalisePath(path) { return path.replace(/^(\.\/)+/, ''); } diff --git a/src/helper/is-object.js b/src/helper/is-object.js new file mode 100644 index 0000000..2edc3fe --- /dev/null +++ b/src/helper/is-object.js @@ -0,0 +1,2 @@ +/** True for a plain object value: not null and not an array. */ +export const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); diff --git a/src/helper/quant-meta.js b/src/helper/quant-meta.js index 6fd98b1..d960f6a 100644 --- a/src/helper/quant-meta.js +++ b/src/helper/quant-meta.js @@ -1,3 +1,5 @@ +import { isObject } from './is-object.js'; + // Validation of the kv_stores section of quant/meta.json. The messages match // the ones the Quant portal shows, so a local run and a deploy agree. @@ -8,7 +10,6 @@ const INDEX_KEYS = ['name', 'collection', 'fields', 'sortBy', 'unique', 'count'] const RESERVED_FIELDS = ['__proto__', 'constructor', 'prototype']; const SHA256 = '{sha256}'; -const isObject = value => value !== null && typeof value === 'object' && !Array.isArray(value); const bytesOf = value => Buffer.byteLength(JSON.stringify(value)); const fail = message => { throw new Error(message); }; const onlyKeys = (value, allowed) => Object.keys(value).every(key => allowed.includes(key)); diff --git a/src/serve/inline.js b/src/serve/inline.js index 347b934..53af2a1 100644 --- a/src/serve/inline.js +++ b/src/serve/inline.js @@ -8,6 +8,7 @@ import fs from 'node:fs'; import path from 'node:path'; import mime from 'mime-types'; +import { isObject } from '../helper/is-object.js'; const TOKEN = /{{QUANT_FN:(.*?)}}/gi; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -26,7 +27,6 @@ const REFUSALS = { filter: 'Direct execution of filter function is disallowed.' }; -const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); function settingsObject(settings, source) { if (Array.isArray(settings) && settings.length === 0) return {}; diff --git a/src/serve/kv/ledgers.js b/src/serve/kv/ledgers.js index 3d6a035..7dfe1f9 100644 --- a/src/serve/kv/ledgers.js +++ b/src/serve/kv/ledgers.js @@ -11,10 +11,10 @@ import fs from 'node:fs'; import path from 'node:path'; import { storeId } from '../../helper/quant-meta.js'; +import { isObject } from '../../helper/is-object.js'; export const LEDGERS_FILE = 'quant/kv-ledgers.json'; -const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); function readText(file) { try { diff --git a/src/serve/kv/remote.js b/src/serve/kv/remote.js index 50665d2..4c6caa3 100644 --- a/src/serve/kv/remote.js +++ b/src/serve/kv/remote.js @@ -8,6 +8,7 @@ import { createLocalKv } from '@quantcdn/edge-kv'; import { createPortalClient, portalOptions, requestBudget, START_REQUEST_LIMIT } from '../portal/client.js'; import { findStore, readStore } from '../portal/kv-remote.js'; +import { isObject } from '../../helper/is-object.js'; export const REMOTE_MAX_KEYS = 10000; export const REMOTE_MAX_REQUESTS = START_REQUEST_LIMIT; @@ -21,7 +22,7 @@ const UNUSED_STATES = ['FAILED', 'DROPPING']; * error, conflicts, job) are dropped. */ export function toLocalIndex(def) { - if (def === null || typeof def !== 'object' || Array.isArray(def)) return null; + if (!isObject(def)) return null; const out = { name: def.name, collection: def.collection, diff --git a/src/serve/portal/pull-rules.js b/src/serve/portal/pull-rules.js index 9145ed8..af56634 100644 --- a/src/serve/portal/pull-rules.js +++ b/src/serve/portal/pull-rules.js @@ -9,6 +9,8 @@ import { PortalError, RequestLimitError } from './client.js'; import { orderRules } from '../rules.js'; +import { ruleLabel } from '../rule-actions.js'; +import { isObject } from '../../helper/is-object.js'; export const PARTIAL_WARNING = 'Some rule types are not in the portal response for this organisation, so they do not run locally.'; export const HEADERS_SCOPE_WARNING = 'QUANT_API_TOKEN cannot read project headers (projects:read), so the portal\'s global headers are not used.'; @@ -29,7 +31,6 @@ const TRUE_TEXT = ['true', '1']; const NUMERIC_TEXT = /^[ \t\n\r\v\f]*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?[ \t\n\r\v\f]*$/; const INT_LIMIT = 2 ** 63; -const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); const projectPath = ({ customer, project }, leaf) => `/organizations/${encodeURIComponent(customer)}/projects/${encodeURIComponent(project)}/${leaf}`; @@ -208,8 +209,6 @@ export async function pullGlobalHeaders(client, { customer, project, warn = () = } } -const ruleLabel = (rule) => rule.name || rule.id; - function conditionWarning(rule) { const names = rule.unchecked.join(', '); const noun = rule.unchecked.length === 1 ? 'condition is' : 'conditions are'; diff --git a/src/serve/project-config.js b/src/serve/project-config.js index bcacbe2..7265973 100644 --- a/src/serve/project-config.js +++ b/src/serve/project-config.js @@ -14,6 +14,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { isObject } from '../helper/is-object.js'; export const SERVE_CONFIG = 'quant/serve.json'; @@ -28,7 +29,6 @@ const PROTECTED = ['quant-server', 'age']; const AUTH_BODY = 'Authentication required'; const REDIRECT_CACHE = 'public, max-age=300, s-maxage=86400'; -const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); const isText = (value) => typeof value === 'string' && value !== ''; function globalHeaders(value, source, warnings) { diff --git a/src/serve/rule-actions.js b/src/serve/rule-actions.js index b7c4a06..e38d0fa 100644 --- a/src/serve/rule-actions.js +++ b/src/serve/rule-actions.js @@ -8,7 +8,7 @@ import { basicAuthResponse } from './project-config.js'; import { PROXY_RESPONSE_DROP, proxyBody, proxyRequestHeaders, withoutHeaders } from './forward.js'; -import { ruleCaptures } from './rules.js'; +import { listsStatus, ruleCaptures } from './rules.js'; const SUPPORTED = new Set(['edge_fn', 'edge_auth', 'edge_filter', 'redirect', 'headers', 'custom_response', 'auth', 'error_page', 'proxy']); // Headers that a headers rule cannot set. @@ -177,8 +177,6 @@ async function forward(ctx, config, target) { return new Response(upstream.body, { status: upstream.status, statusText: upstream.statusText, headers: result }); } -const listsStatus = (list, status) => Array.isArray(list) && list.map(String).includes(String(status)); - // With static error pages on, a listed origin status gets the published error page. function withErrorPage(state, ctx, config, response) { if (!config.static_error_page || !listsStatus(config.static_error_page_status_codes, response.status)) return response; diff --git a/src/serve/rules.js b/src/serve/rules.js index 89d752f..a02eaaf 100644 --- a/src/serve/rules.js +++ b/src/serve/rules.js @@ -89,15 +89,12 @@ export function matchFilter(rules, ctx) { return rules.find(rule => rule.action === 'edge_filter' && ruleMatches(rule, ctx)) || null; } -// A status_codes value that is not a list lists no status. -function listsStatus(rule, status) { - const codes = rule.config?.status_codes; - return Array.isArray(codes) && codes.map(String).includes(String(status)); -} +/** True when `list` is a list of status codes that holds `status`. A value that is not a list lists no status. */ +export const listsStatus = (list, status) => Array.isArray(list) && list.map(String).includes(String(status)); /** The first enabled error page rule for the status that matches, or null. */ export function matchErrorPage(rules, ctx, status) { - return rules.find(rule => rule.action === 'error_page' && listsStatus(rule, status) && ruleMatches(rule, ctx)) || null; + return rules.find(rule => rule.action === 'error_page' && listsStatus(rule.config?.status_codes, status) && ruleMatches(rule, ctx)) || null; } /** diff --git a/src/serve/seed.js b/src/serve/seed.js index 7421009..1bccbd7 100644 --- a/src/serve/seed.js +++ b/src/serve/seed.js @@ -9,6 +9,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import { appendRevealedKey } from './kv/state.js'; +import { isObject } from '../helper/is-object.js'; export const MAX_FIXTURE_BYTES = 1024 * 1024; export const MAX_FIXTURE_ENTRIES = 5000; @@ -17,7 +18,6 @@ const USER_NAME = /\{user_name\}/g; const DATE_OFFSET = /^(now|[+-]\d+[mhd])$/; const UNIT_MS = { m: 60 * 1000, h: 60 * 60 * 1000, d: 24 * 60 * 60 * 1000 }; -const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); const digest = (value) => crypto.createHash('sha256').update(value).digest('hex'); const randomHex = (bytes) => crypto.randomBytes(bytes).toString('hex'); From da723607a8c1184ef6821767e3e70fdf7975dda5 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 23:22:05 +1000 Subject: [PATCH 29/32] docs(serve): drop an internal name from a code comment --- src/helper/functions-manifest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helper/functions-manifest.js b/src/helper/functions-manifest.js index 40ab6c8..1b2e661 100644 --- a/src/helper/functions-manifest.js +++ b/src/helper/functions-manifest.js @@ -1,7 +1,7 @@ /** * The edge function manifest schema (quant/edge-functions.json). * - * The same rules as the Quant portal (App\Helpers\TemplateManifestPaths), so a + * The same rules as the Quant portal, so a * manifest that passes here passes there, and the reverse. Every entry names * its function by an explicit uuid: no tool derives one. */ From 87e22a46f0ec8d1b548948235fbb03140e7a48bd Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 4 Nov 2026 23:22:05 +1000 Subject: [PATCH 30/32] release: 7.4.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d7b0dc..e01f6df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quantcdn/quant-cli", - "version": "7.3.0", + "version": "7.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@quantcdn/quant-cli", - "version": "7.3.0", + "version": "7.4.0", "license": "ISC", "dependencies": { "@clack/core": "^1.0.0", diff --git a/package.json b/package.json index 05b5d28..dcaa9f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@quantcdn/quant-cli", - "version": "7.3.0", + "version": "7.4.0", "description": "Deploy tools for QuantCDN", "type": "module", "main": "cli.js", From 6f4f9bfe150826f5e4fccb789f867d62297910c6 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Thu, 5 Nov 2026 06:25:40 +1000 Subject: [PATCH 31/32] fix(serve): find the --kv-remote store by user_name or store id The portal store list can send a display name in name (for example "org/project: store"). Match the meta.json name on user_name first, then on the project store id, then on name. --- src/serve/portal/kv-remote.js | 14 ++++++++++++-- tests/fixtures/fake-portal/fixtures.json | 6 ++++-- tests/unit/serve/kv-remote-read.test.mjs | 11 +++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/serve/portal/kv-remote.js b/src/serve/portal/kv-remote.js index e31d8c8..eddeb0f 100644 --- a/src/serve/portal/kv-remote.js +++ b/src/serve/portal/kv-remote.js @@ -1,3 +1,4 @@ +import { storeId } from '../../helper/quant-meta.js'; import { PortalError, RequestLimitError } from './client.js'; const PAGE_LIMIT = 1000; @@ -11,12 +12,21 @@ const storePath = ({ customer, project }, leaf = '') => const notLinked = name => new PortalError(`The KV store ${name} is not linked to this project.`, 404); const scopeInfo = name => ({ scope: READ_SCOPE, what: `KV store ${name}` }); +// The list row of the store: by user_name (the meta.json name), then by the +// project store id, then by name. The portal can send a display name in name. +function matchStore(stores, { customer, project, name }) { + const id = storeId(customer, project, name); + return stores.find(store => store.user_name === name) + ?? stores.find(store => store.id === id) + ?? stores.find(store => store.name === name); +} + export async function findStore(client, { customer, project, name }) { const { status, body } = await client.get(storePath({ customer, project }), undefined, scopeInfo(name)); const stores = status === 404 || !Array.isArray(body) ? [] : body; - const found = stores.find(store => store && store.name === name); + const found = matchStore(stores.filter(Boolean), { customer, project, name }); if (!found) throw notLinked(name); - return { id: found.id, name: found.name, ...([1, 2].includes(found.format) ? { format: found.format } : {}) }; + return { id: found.id, name, ...([1, 2].includes(found.format) ? { format: found.format } : {}) }; } async function fetchOk(name, send) { diff --git a/tests/fixtures/fake-portal/fixtures.json b/tests/fixtures/fake-portal/fixtures.json index 509da75..9473aef 100644 --- a/tests/fixtures/fake-portal/fixtures.json +++ b/tests/fixtures/fake-portal/fixtures.json @@ -166,8 +166,10 @@ }, "stores": [ { - "id": "abc", - "name": "cache" + "id": "acme-site-cache", + "name": "acme/site: cache", + "user_name": "cache", + "linked": false } ], "indexes": [], diff --git a/tests/unit/serve/kv-remote-read.test.mjs b/tests/unit/serve/kv-remote-read.test.mjs index b3106ec..906c65c 100644 --- a/tests/unit/serve/kv-remote-read.test.mjs +++ b/tests/unit/serve/kv-remote-read.test.mjs @@ -55,6 +55,17 @@ describe('remote KV read', () => { expect(calls[0].route).to.equal('/'); }); + it('matches the user_name of a row with a display name, and accepts linked false with no format', async () => { + const row = { id: 'org-proj-sessions', name: 'org/proj: sessions', user_name: 'sessions', linked: false }; + const other = { id: 'org-proj-local', name: 'sessions', user_name: 'local', linked: false }; + expect(await findStore(portal({ stores: [other, row] }).client, CTX)).to.deep.equal({ id: 'org-proj-sessions', name: 'sessions' }); + }); + + it('matches the project store id when the row has no user_name', async () => { + const row = { id: 'org-proj-sessions', name: 'org/proj: sessions' }; + expect(await findStore(portal({ stores: [row] }).client, CTX)).to.deep.equal({ id: 'org-proj-sessions', name: 'sessions' }); + }); + it('passes the store format through when the list reports 1 or 2', async () => { expect(await findStore(portal({ stores: [{ id: 'abc', name: 'sessions', format: 1 }] }).client, CTX)).to.deep.equal({ id: 'abc', name: 'sessions', format: 1 }); expect(await findStore(portal({ stores: [{ id: 'abc', name: 'sessions', format: 'x' }] }).client, CTX)).to.deep.equal({ id: 'abc', name: 'sessions' }); From 9f20f558d09e2a225b71699683c5e01aaf96fa0a Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Thu, 5 Nov 2026 06:54:18 +1000 Subject: [PATCH 32/32] feat(serve): print the count of pulled global headers at start Add a docs note that the trailing-slash setting and the edge cache settings are not pulled. --- README.md | 5 +++-- src/commands/serve.js | 6 ++++-- tests/unit/serve/phase3-e2e.test.mjs | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 080c841..051a265 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ quant serve dist --pull-rules --kv-remote sessions --org acme --project marketin The pulled rules run in the order of the edge, with the rules of `quant/edge-rules.json`. A local rule replaces the portal rule that a deploy of the same namespace and rule ID owns. All other portal rules stay in their order: also a deployed rule that the local file no longer lists, and the deployed rules of other namespaces. Each local rule then goes before the first portal rule with a higher weight. A deploy also creates or updates only the rules of its own namespace and removes no rule, so this is the order that the edge has after a deploy of the same file when the portal rules are in weight order. Without `quant/edge-rules.json`, the portal rules run unchanged. -The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules that run with the portal rules. +The start-up output shows `Pulled portal rules ( from quant/edge-rules.json).`. `` is the number of local rules that run with the portal rules. If the portal project has global headers, the next line is `Using portal global headers.`. The line shows the count only, not the names or values. Actions that run locally: redirect (with `$1` to `$5`), headers, custom response, basic auth, error page, proxy, and edge function, auth function and filter rules. A rule with a cookie condition runs only when the request sends a cookie of exactly that name. @@ -508,7 +508,8 @@ These do not run locally: Other points: - Some organisations (for example trial organisations) do not get every rule type in the portal list. `quant serve` then prints `Some rule types are not in the portal response for this organisation, so they do not run locally.`, and the missing rules do not run. -- Basic authentication for the whole site and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. +- Basic authentication for the whole site and the trailing-slash setting have no portal read route. They come only from `quant/serve.json`. For example, if the project redirects `/app/` to `/app` at the edge, set `"trailingSlash": "redirect"` in `quant/serve.json`. +- Static responses from `quant serve` have `cache-control: no-store`. The edge cache settings are not pulled. - The `globalHeaders` of `quant/serve.json` replace a portal global header of the same name, with one warning for each name. - A pulled function rule runs only when its function uuid is in `quant/edge-functions.json`. The function code is not pulled. diff --git a/src/commands/serve.js b/src/commands/serve.js index c990575..157ee94 100644 --- a/src/commands/serve.js +++ b/src/commands/serve.js @@ -234,7 +234,9 @@ function ledgerLines(ledgers, clock) { } function pullLines(pull) { - return pull ? [`Pulled ${pull.count} portal rules (${pull.local} from quant/edge-rules.json).`] : []; + if (!pull) return []; + const headers = pull.headers ? [`Using ${pull.headers} portal global header${pull.headers === 1 ? '' : 's'}.`] : []; + return [`Pulled ${pull.count} portal rules (${pull.local} from quant/edge-rules.json).`, ...headers]; } function clockLines(clock) { @@ -484,7 +486,7 @@ async function pulledProject(options, { env = process.env, fetchImpl, budget } = // The start-up summary of a pull: the pulled rules and the local rules merged in. function pullSummary(pulled, server) { - return pulled ? { count: pulled.count, local: server.summary.localRules } : null; + return pulled ? { count: pulled.count, local: server.summary.localRules, headers: Object.keys(pulled.headers ?? {}).length } : null; } function withPull(options, pulled) { diff --git a/tests/unit/serve/phase3-e2e.test.mjs b/tests/unit/serve/phase3-e2e.test.mjs index 0688ce1..e854c63 100644 --- a/tests/unit/serve/phase3-e2e.test.mjs +++ b/tests/unit/serve/phase3-e2e.test.mjs @@ -173,6 +173,7 @@ describe('quant serve phase 3 end to end', function () { it('prints no token or secret, and writes no snapshot value or token to the project', async () => { const printed = site.output(); expect(printed).to.include('Pulled 9 portal rules (4 from quant/edge-rules.json).'); + expect(printed).to.include('Using 1 portal global header.'); for (const secret of [TOKEN, BASIC_PASSWORD, INJECT_SECRET, PORTAL_VALUE, LOCAL_WRITE]) expect(printed).to.not.include(secret); await site.stop(); expect(fs.existsSync(path.join(root, '.quant'))).to.equal(true);