From 6e83f29060af33754cf27a4bd96e71d168381065 Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:32:56 +0800 Subject: [PATCH 1/6] docs: add first SQL walkthrough with source consistency checks The quickstart shows three SQL statements with psql prompts and sample output, but never explains them statement by statement, and a separate explanation can silently drift from the text it quotes. Add a companion guide for exactly those three existing statements and an offline checker that reports when the quoted statements drift. - docs/guide/first-three-sql.md: reading order for CREATE TABLE, INSERT and SELECT, with prerequisites, labelled expected results, common misunderstandings, pinned source links and five next-step entries - docs/guide/01-quickstart.md: one link to the guide; the existing SQL and sample output are unchanged - tools/sql_guide/{check.py,examples.json,README.md}: standard-library only, offline, never executes SQL; text and JSON output; exit code 0 on a full match, 1 on document or locator problems, 2 on invalid arguments or invalid manifest - tools/sql_guide/tests/test_check.py: 28 tests covering the happy path, string drift, whitespace inside literals, missing and ambiguous source statements, broken or duplicate guide markers, invalid manifests and the JSON report The checker validates reference consistency only. It does not claim the explanation is technically correct, and it does not run any SQL. --- docs/guide/01-quickstart.md | 1 + docs/guide/first-three-sql.md | 186 ++++++++ tools/sql_guide/README.md | 99 ++++ tools/sql_guide/check.py | 701 ++++++++++++++++++++++++++++ tools/sql_guide/examples.json | 24 + tools/sql_guide/tests/test_check.py | 417 +++++++++++++++++ 6 files changed, 1428 insertions(+) create mode 100644 docs/guide/first-three-sql.md create mode 100644 tools/sql_guide/README.md create mode 100644 tools/sql_guide/check.py create mode 100644 tools/sql_guide/examples.json create mode 100644 tools/sql_guide/tests/test_check.py diff --git a/docs/guide/01-quickstart.md b/docs/guide/01-quickstart.md index 381bd7e..2b237bd 100644 --- a/docs/guide/01-quickstart.md +++ b/docs/guide/01-quickstart.md @@ -322,6 +322,7 @@ test=> select * from foo; ``` +> 上面这三条语句(建表、插入、查询)的逐步解释、前置条件和常见误解,见[读懂第一条 SQL:建表、插入与查询](first-three-sql.md)。 - **结语** 本文档只是给用户一个简单的指引,演示如何从源码开始,一步一步搭建一个完整的OpenTenBase集群,后续会有更多的文章来介绍OpenTenBase的特性使用,优化,问题定位等内容。 diff --git a/docs/guide/first-three-sql.md b/docs/guide/first-three-sql.md new file mode 100644 index 0000000..408d2ac --- /dev/null +++ b/docs/guide/first-three-sql.md @@ -0,0 +1,186 @@ +# 读懂第一条 SQL:建表、插入与查询 + +> 本文伴读[快速入门](01-quickstart.md)的“使用”小节。那里给出了三条 SQL 和它们的执行输出,但没有逐条说明每条语句在做什么。本文按“建表 → 插入 → 查询”的顺序把这三条拆开讲清楚。 +> +> 本文不新增 SQL,也不改写原文中的 SQL。文中标注为“来源文档展示”的输出,均为原文照录,不是本文作者的实测结果。 + +## 先分清三种提示符 + +原文的代码块里混着三类行:shell 命令、SQL 语句、执行输出。它们的区别看行首: + +| 行首 | 是什么 | 原文例子 | +|---|---|---| +| `$` | 操作系统 shell 提示符,后面是 shell 命令 | `$ psql -h ${CoordinateNode_IP} ...` | +| `postgres=#` | psql 提示符,当前连的是 `postgres` 库;`#` 表示超级用户 | `postgres=# create database test;` | +| `test=>` | psql 提示符,执行 `\c test test` 之后已切到 `test` 库;`>` 表示普通用户 | `test=> select * from foo;` | +| 无提示符 | 上一条语句的执行输出 | `CREATE TABLE`、`(2 rows)` | + +复制示例时只复制提示符后面的部分。把 `test=>` 一起粘进 psql,会得到一个语法错误,而不是执行结果。 + +## 前置条件 + +三条语句依赖原文前一段的准备工作,按顺序是: + +1. 集群已启动,并且能通过 CN(协调节点)连接。 +2. 已建库和用户,并把库的属主交给该用户:原文执行 `create database test;`、`create user test ...`、`alter database test owner to test;`。 +3. 已切换到目标库:原文执行 `\c test test`,之后提示符才从 `postgres=#` 变成 `test=>`。 + +这也是下文三条语句的提示符都是 `test=>` 的原因:它们是在 `test` 库里、以普通用户 `test` 的身份执行的。 + +> 原文的建用户示例使用了一个演示用的简单口令。它只适合一次性演示环境,不要复制到任何真实部署中;真实环境请使用符合口令策略的强口令。 + +## 第一步:建表 + +**用途**:在 `test` 库里建一张名为 `foo` 的表,并指定按 `id` 列分布数据。 + +**原文语句**: + + +```sql +create table foo(id bigint, str text) distribute by shard(id); +``` + +**逐段拆解**: + +| 片段 | 含义 | +|---|---| +| `create table foo` | 建表,表名 `foo` | +| `(id bigint, str text)` | 两个列:`id` 是 64 位整数,`str` 是变长文本 | +| `distribute by shard(id)` | OpenTenBase 的分布子句,按 `id` 列的值决定每一行落到哪个 DN(数据节点) | + +**关于 `distribute by shard(id)`**:这是 OpenTenBase 在标准 SQL 之外提供的语法,用来描述数据在多个 DN 之间怎么分。理解它时有三个容易过头的地方: + +- 它不是 PostgreSQL 或 MySQL 的通用写法,照搬到别的数据库会报错。 +- “按 `id` 分布”不等于“每个 `id` 独占一个节点”,也不等于“两行数据一定落在两个不同的 DN”。 +- 不能只凭插入的行数推断分布结果。某一行实际落在哪个 DN,取决于分布键的取值和节点数量。 + +**前置条件**:已按上文完成建库、建用户并切换到 `test` 库。 + +**来源文档展示**:紧跟这条语句,原文给出的标签是 `CREATE TABLE`。(原文照录,本文未执行。) + +**常见误解**: + +- 把 `distribute by shard(id)` 理解成索引或分区。它是数据的分布方式,不创建索引。 +- 以为这段建表语句能直接在单机 PostgreSQL 上运行。它依赖 OpenTenBase 的集群能力。 +- 以为两行数据必然分散在不同节点。分布是按下标值计算的,和插入了几行没有关系。 + +**来源**:`docs/guide/01-quickstart.md`(固定提交 `4eaeba90188926ddf4da23b74775cdf150ffdfe8`),“使用”小节代码块中,`\c test test` 之后的第一条 `test=>` 语句。[在固定提交上查看](https://github.com/OpenTenBase/docs/blob/4eaeba90188926ddf4da23b74775cdf150ffdfe8/docs/guide/01-quickstart.md) + +## 第二步:插入 + +**用途**:向 `foo` 一次插入两行数据。 + +**原文语句**: + + +```sql +insert into foo values(1, 'tencent'), (2, 'shenzhen'); +``` + +**逐段拆解**: + +| 片段 | 含义 | +|---|---| +| `insert into foo` | 向 `foo` 表插入 | +| `values(1, 'tencent'), (2, 'shenzhen')` | 两组括号就是两行,逗号分隔 | +| `1`、`2` | 对应 `id` 列(整数,不加引号) | +| `'tencent'`、`'shenzhen'` | 对应 `str` 列(字符串,用单引号) | + +**为什么不写列清单**:这条语句省略了 `(id, str)`。省略时,`values` 里的值按建表时的列顺序依次匹配,所以第一个值进 `id`,第二个值进 `str`。写全列清单更不容易出错,但原文示例没有写,本文不改写它。 + +**前置条件**:`foo` 表已存在,即第一步已成功。否则插入会失败。 + +**来源文档展示**:紧跟这条语句,原文给出的标签是 `COPY 2`。(原文照录。) + +> 说明:本文只记录原文写了什么,不对这个标签与当前版本的实际输出是否一致作判断。如果你在自己的环境里执行,请以实际输出为准。 + +**常见误解**: + +- 把两组括号当成一行两列。这里一个括号是一行,两个括号是两行,每行两列。 +- 用双引号写字符串。SQL 里双引号是标识符,字符串必须用单引号。 +- 以为值的个数可以随意。`values` 每组的值的数量必须和表的列数对应。 + +**来源**:`docs/guide/01-quickstart.md`(固定提交 `4eaeba90188926ddf4da23b74775cdf150ffdfe8`),“使用”小节代码块中,建表语句之后的第一条 `test=>` 语句。[在固定提交上查看](https://github.com/OpenTenBase/docs/blob/4eaeba90188926ddf4da23b74775cdf150ffdfe8/docs/guide/01-quickstart.md) + +## 第三步:查询 + +**用途**:把 `foo` 里的数据读出来,确认前两步的结果。 + +**原文语句**: + + +```sql +select * from foo; +``` + +**逐段拆解**: + +| 片段 | 含义 | +|---|---| +| `select` | 查询 | +| `*` | 返回表里的全部列(这里就是 `id` 和 `str` 两列) | +| `from foo` | 从 `foo` 表读取 | + +**预期结果**(来源文档展示):原文给出的结果是一个两列表格,内容如下。 + +| id | str | +|---|---| +| 1 | tencent | +| 2 | shenzhen | + +这里必须说清三点: + +- 本文没有实际执行这条查询,上表是**预期结果**,来源是原文展示。 +- `select *` 返回的是全部列,不是“全部数据”这个意义上的一切;本例恰好返回全部两行。 +- 这条语句**没有 `ORDER BY`**,所以不保证任何返回顺序。原文展示的 `1`、`2` 顺序只是那次展示的顺序,不是可以依赖的契约。需要固定顺序时必须显式写 `ORDER BY`。 + +**前置条件**:`foo` 表已存在且有数据,即前两步都已成功。 + +**来源文档展示**:原文在结果表格后给出 `(2 rows)`,表示返回两行。(原文照录。) + +**常见误解**: + +- 以为没有 `ORDER BY` 也会按插入顺序返回。不保证。 +- 以为 `select *` 是推荐写法。列结构变化会让下游消费方受影响,正式查询建议写出需要的列名。 +- 以为查询结果为空的表和报错是一回事。空表返回 0 行,不是错误。 + +**来源**:`docs/guide/01-quickstart.md`(固定提交 `4eaeba90188926ddf4da23b74775cdf150ffdfe8`),“使用”小节代码块中的第三条 `test=>` 语句。[在固定提交上查看](https://github.com/OpenTenBase/docs/blob/4eaeba90188926ddf4da23b74775cdf150ffdfe8/docs/guide/01-quickstart.md) + +## 把三条连起来看 + +| 顺序 | 语句作用 | 依赖 | 来源文档展示的输出 | +|---|---|---|---| +| 第一步 | 建表 `foo`,两列,按 `id` 分布 | 已建库、建用户、切到 `test` 库 | `CREATE TABLE` | +| 第二步 | 插入两行数据 | `foo` 已存在 | `COPY 2` | +| 第三步 | 读出全部列 | `foo` 已存在且有数据 | 两行结果、`(2 rows)` | + +三步是严格的先后关系:建表失败,插入必然失败;插入失败,查询会得到空结果而不是两行。 + +## 校验这三条语句有没有和原文走样 + +`docs/guide/first-three-sql.md`(也就是本文)里的三条 SQL 是逐字引用原文的。为了避免日后维护时原文改了、伴读没跟上,仓库里带了一个离线校验工具: + +``` +python3 tools/sql_guide/check.py --root . +``` + +它只做一件事:检查本文中标记的三条语句与 `docs/guide/01-quickstart.md` 里的原文是否仍然逐字一致。它不联网、不连数据库、不执行任何 SQL。 + +它的通过只说明**引用没有漂移**,不说明本文的技术解释正确,也不说明这三条语句在你的环境里能跑通。使用方法、退出码和已知限制见仓库内 `tools/sql_guide/README.md`。 + +## 下一步读什么 + +按下面的顺序继续,每条都说明了它适合解决什么问题: + +1. [快速入门](01-quickstart.md):回到原文,确认集群部署步骤和这三条语句的上下文。 +2. [应用接入指南](02-access.md):用 Java、C、shell、Python、PHP 或 Go 连接 OpenTenBase。 +3. [基本使用](03-basic-use.md):继续学建库、建表、数据导入和更复杂的查询。 +4. [用户管理](07-user-rights.md):理解本文示例里的 `test` 用户、库属主和权限之间的关系。 +5. [如何贡献 OpenTenBase](../contribution/how-to.md):发现文档问题或有改进建议时,提 issue 或 PR 的流程。 + +## 本文的边界 + +- 三条语句逐字取自来源文档,本文不改写,也不新增第四条。 +- 本文未在真实环境执行这些语句;标注为“来源文档展示”的输出是原文照录,标注为“预期结果”的内容由原文展示推导而来。 +- 本文不涉及性能调优、分布键的选择策略、权限模型的细节。 +- 引用一致性可以借助上述工具复查;解释是否准确仍需人工评审。如果发现解释与你的实际行为不一致,欢迎按第 5 条入口反馈。 diff --git a/tools/sql_guide/README.md b/tools/sql_guide/README.md new file mode 100644 index 0000000..233a358 --- /dev/null +++ b/tools/sql_guide/README.md @@ -0,0 +1,99 @@ +# SQL 伴读来源一致性校验 + +这个目录里只有一个工具,用处很窄:检查 `docs/guide/first-three-sql.md` 中引用的三条 SQL,是否仍与 `docs/guide/01-quickstart.md` 里的原文逐字一致。 + +## 为什么需要它 + +伴读手册的三条语句是从快速入门里逐字复制的。只要原文改了、伴读没跟上,读者就会看到一份“看起来对、实际上是旧版本”的解释。人工复查容易漏,这个工具把这件事变成一条命令。 + +它只回答“引用有没有走样”,不回答“解释对不对”,也不回答“SQL 能不能跑通”。 + +## 用法 + +在仓库根目录执行: + +```bash +python3 tools/sql_guide/check.py --root . +``` + +输出示例: + +``` +sql-guide source consistency check + manifest: tools/sql_guide/examples.json + source: docs/guide/01-quickstart.md + guide: docs/guide/first-three-sql.md + +PASS create-foo-table guide:40 <- source:312 +PASS insert-foo-rows guide:77 <- source:314 +PASS select-foo-all guide:113 <- source:316 + +summary: 3 examples, 3 passed, 0 failed +``` + +机器可读输出: + +```bash +python3 tools/sql_guide/check.py --root . --format json +``` + +JSON 包含 `schema_version`、`summary`、`files`、`results`、`issues`。出现不一致时,`results[].expected` 与 `results[].actual` 会给出原文与伴读两侧的语句(超过 160 字符截断)。报告不包含任何其他文件内容,也不包含口令。 + +## 退出码 + +| 退出码 | 含义 | +|---|---| +| `0` | 配置合法,三个示例全部与来源一致 | +| `1` | 文档缺失、定位失败、示例重复,或语句与来源不一致 | +| `2` | 命令行参数错误,或 `examples.json` 缺失/不是合法 JSON/结构不合法 | + +`1` 与 `2` 的区分很重要:`1` 说明文档需要修,`2` 说明工具配置本身需要修。 + +## 定位方式 + +工具不依赖行号,因为行号会随文档编辑漂移。它依赖三件事: + +1. **来源侧**:`examples.json` 指定来源文件和每条语句的开头(`statement_prefix`)。工具只扫描代码围栏内部、以 `postgres=#` 或 `test=>` 提示符开头的行,提示符之后的部分才是语句。围栏内的命令行、元命令和执行输出都不会被当成语句。 +2. **伴读侧**:每条语句上方有一个 HTML 注释标记,标记与紧邻的 `sql` 围栏绑定: + + ````markdown + + ```sql + create table foo(id bigint, str text) distribute by shard(id); + ``` + ```` + + 标记与围栏之间只允许空行。 +3. **唯一性**:每个 ID 在伴读里必须恰好出现一次,在来源里必须恰好匹配一条语句。匹配到两条以上时,工具报“歧义”并失败,不会自己挑一条。 + +## 比对规则 + +只做两件事:去掉已声明的提示符,去掉行首行尾空白。此外一律逐字比较: + +- 不折叠字符串内部的空格(`'shenzhen'` 与 `'shen zhen'` 视为不同); +- 不忽略大小写; +- 不做标点归一化,不替换全角/半角。 + +宁可误报,也不做模糊匹配——模糊匹配会让“已经改坏了但看起来差不多”的情况通过。 + +## 已知限制 + +- 只支持本项目约定的三个单行示例,不是通用 Markdown 解析器,也不是 SQL 解析器。 +- 只识别 `postgres=#` 与 `test=>` 两种提示符,且必须在代码围栏内。 +- 只比较语句文本。语句的**技术解释**是否正确、能否在某个环境执行、性能如何,都不在检查范围内。 +- 不联网、不连接数据库、不执行任何 SQL、不修改仓库文件。 +- 通过检查不代表文档整体正确,只代表这三条引用没有漂移。 + +## 测试 + +```bash +python3 -m unittest discover -s tools/sql_guide/tests -v +``` + +测试覆盖:正常通过、字符串字面量被改、内部空格差异、分号缺失、来源文件缺失、来源语句缺失、来源出现两条相同候选、伴读标记缺失/重复/后接非 SQL 围栏/多行代码块、清单非法 JSON/缺键/多键/版本不支持、JSON 报告可解析且计数一致、`--help` 返回 0,以及对本仓库真实文档的端到端检查。 + +## 新增一个示例 + +1. 在 `examples.json` 的 `examples` 里加一项:唯一 `id`,以及来源语句的开头 `statement_prefix`(要足够独特,避免匹配到多条)。 +2. 在伴读文档中,把语句放进以 `sql` 标注的代码围栏,并在围栏上一行加 ``。 +3. 运行 `python3 tools/sql_guide/check.py --root .`,确认新示例为 `PASS`。 diff --git a/tools/sql_guide/check.py b/tools/sql_guide/check.py new file mode 100644 index 0000000..64a1c2f --- /dev/null +++ b/tools/sql_guide/check.py @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Offline source-consistency checker for the first-three-SQL walkthrough. + +The checker answers exactly one question: are the three SQL statements quoted +in docs/guide/first-three-sql.md still byte-for-byte identical to the three +statements shown in docs/guide/01-quickstart.md? + +Deliberate limits: + +* standard library only, no third-party imports; +* reads local files, never opens a network connection; +* never executes SQL and never starts a database; +* only understands the narrow Markdown subset described in README.md; +* proves reference consistency, not technical correctness. + +Exit codes: 0 all examples match, 1 document/locator/mismatch problem, +2 invalid CLI arguments or invalid manifest. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple + +EXIT_OK = 0 +EXIT_MISMATCH = 1 +EXIT_USAGE = 2 + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST = Path("tools") / "sql_guide" / "examples.json" + +# A fenced code block: at most three spaces of indent, then ``` or ~~~ (3+), +# then an optional info string. +FENCE_RE = re.compile(r"^(?P[ \t]{0,3})(?P`{3,}|~{3,})(?P.*)$") + +# The guide binds an ID to the next fenced block with this HTML comment: +# +MARKER_RE = re.compile( + r"" +) + +PREVIEW_LIMIT = 160 + + +class ManifestError(Exception): + """The manifest is missing, unreadable, or structurally invalid.""" + + +@dataclass(frozen=True) +class CodeLine: + number: int + text: str + + +@dataclass(frozen=True) +class Fence: + start_index: int # 0-based index of the opening fence line + start_line: int # 1-based line number of the opening fence + info: str + lines: Tuple[CodeLine, ...] + closed: bool + + +@dataclass(frozen=True) +class Statement: + line: int + sql: str + + +@dataclass(frozen=True) +class Example: + id: str + statement_prefix: str + + +@dataclass(frozen=True) +class Manifest: + schema_version: int + source_document: str + guide_document: str + allowed_prompts: Tuple[str, ...] + guide_marker: str + examples: Tuple[Example, ...] + + +@dataclass +class Binding: + """One ID marker found in the guide, resolved to a SQL line when possible.""" + + line: int + sql: Optional[str] = None + sql_line: Optional[int] = None + problem: str = "" + + +@dataclass +class Result: + example_id: str + passed: bool + reason: str = "" + source_line: Optional[int] = None + guide_line: Optional[int] = None + expected: Optional[str] = None + actual: Optional[str] = None + + +@dataclass +class Report: + manifest_path: str + source_path: str + guide_path: str + results: List[Result] = field(default_factory=list) + issues: List[str] = field(default_factory=list) + + @property + def total(self) -> int: + return len(self.results) + + @property + def passed(self) -> int: + return sum(1 for item in self.results if item.passed) + + @property + def failed(self) -> int: + return self.total - self.passed + + def exit_code(self) -> int: + if self.issues or self.failed: + return EXIT_MISMATCH + return EXIT_OK + + +# -------------------------------------------------------------------------- +# Markdown scanning +# -------------------------------------------------------------------------- + + +def parse_fences(text: str) -> List[Fence]: + """Return every fenced code block in document order. + + An unclosed fence runs to the end of the document; callers do not treat + that as an error, because the statements we look for are matched by + content, not by fence boundaries. + """ + lines = text.splitlines() + fences: List[Fence] = [] + index = 0 + total = len(lines) + + while index < total: + match = FENCE_RE.match(lines[index]) + if match is None: + index += 1 + continue + + token = match.group("fence") + char = token[0] + length = len(token) + info = match.group("info").strip() + start_index = index + start_line = index + 1 + index += 1 + + body: List[CodeLine] = [] + closed = False + while index < total: + stripped = lines[index].strip() + if len(stripped) >= length and set(stripped) == {char}: + closed = True + index += 1 + break + body.append(CodeLine(index + 1, lines[index])) + index += 1 + + fences.append(Fence(start_index, start_line, info, tuple(body), closed)) + + return fences + + +def extract_source_statements(text: str, prompts: Sequence[str]) -> List[Statement]: + """Pull candidate statements out of fenced blocks in the source document. + + Only lines that start with one of the declared prompts qualify. Everything + else inside a fence -- command output, meta-commands, shell lines -- is + ignored, so no output line can be mistaken for a statement. + """ + ordered = sorted(prompts, key=len, reverse=True) + statements: List[Statement] = [] + + for fence in parse_fences(text): + for code_line in fence.lines: + stripped = code_line.text.strip() + for prompt in ordered: + if not stripped.startswith(prompt): + continue + sql = stripped[len(prompt) :].strip() + if sql: + statements.append(Statement(code_line.number, sql)) + break + + return statements + + +def load_manifest(path: Path) -> Manifest: + """Read and validate the manifest, raising ManifestError on any defect.""" + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + raise ManifestError("manifest not found: {}".format(path)) from None + except OSError as exc: + raise ManifestError("manifest unreadable: {} ({})".format(path, exc)) from None + + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise ManifestError( + "manifest is not valid JSON: {} (line {}, column {})".format( + path, exc.lineno, exc.colno + ) + ) from None + + if not isinstance(data, dict): + raise ManifestError("manifest root must be a JSON object") + + allowed_keys = { + "schema_version", + "source_document", + "guide_document", + "allowed_prompts", + "guide_marker", + "examples", + } + unknown = sorted(set(data) - allowed_keys) + if unknown: + raise ManifestError( + "manifest has unsupported key(s): {}".format(", ".join(unknown)) + ) + missing = sorted(allowed_keys - set(data)) + if missing: + raise ManifestError( + "manifest is missing key(s): {}".format(", ".join(missing)) + ) + + version = data["schema_version"] + if not isinstance(version, int) or isinstance(version, bool): + raise ManifestError("schema_version must be an integer") + if version != SCHEMA_VERSION: + raise ManifestError( + "unsupported schema_version {}, expected {}".format( + version, SCHEMA_VERSION + ) + ) + + for key in ("source_document", "guide_document", "guide_marker"): + if not isinstance(data[key], str) or not data[key].strip(): + raise ManifestError("{} must be a non-empty string".format(key)) + + prompts = data["allowed_prompts"] + if not isinstance(prompts, list) or not prompts: + raise ManifestError("allowed_prompts must be a non-empty list") + for prompt in prompts: + if not isinstance(prompt, str) or not prompt: + raise ManifestError("every allowed_prompts entry must be a non-empty string") + + raw_examples = data["examples"] + if not isinstance(raw_examples, list) or not raw_examples: + raise ManifestError("examples must be a non-empty list") + + examples: List[Example] = [] + seen_ids = set() + for position, item in enumerate(raw_examples, start=1): + if not isinstance(item, dict): + raise ManifestError("examples[{}] must be a JSON object".format(position)) + extra = sorted(set(item) - {"id", "statement_prefix"}) + if extra: + raise ManifestError( + "examples[{}] has unsupported key(s): {}".format( + position, ", ".join(extra) + ) + ) + example_id = item.get("id") + prefix = item.get("statement_prefix") + if not isinstance(example_id, str) or not example_id.strip(): + raise ManifestError("examples[{}].id must be a non-empty string".format(position)) + if not re.fullmatch(r"[A-Za-z0-9_.-]+", example_id): + raise ManifestError( + "examples[{}].id must match [A-Za-z0-9_.-]+, got {!r}".format( + position, example_id + ) + ) + if not isinstance(prefix, str) or not prefix.strip(): + raise ManifestError( + "examples[{}].statement_prefix must be a non-empty string".format(position) + ) + if example_id in seen_ids: + raise ManifestError("duplicate example id: {}".format(example_id)) + seen_ids.add(example_id) + examples.append(Example(example_id, prefix)) + + return Manifest( + schema_version=version, + source_document=data["source_document"], + guide_document=data["guide_document"], + allowed_prompts=tuple(prompts), + guide_marker=data["guide_marker"], + examples=tuple(examples), + ) + + +def bind_guide(text: str, marker: str) -> Tuple[Dict[str, List[Binding]], List[str]]: + """Resolve every `` in the guide to the block after it. + + A binding is only accepted when the marker is followed by nothing but blank + lines and then a single-line ```sql fence. Anything else is reported, never + silently skipped. + """ + lines = text.splitlines() + fences = parse_fences(text) + bindings: Dict[str, List[Binding]] = {} + issues: List[str] = [] + + for index, raw in enumerate(lines): + match = MARKER_RE.search(raw) + if match is None or match.group("key") != marker: + continue + + ident = match.group("ident") + marker_line = index + 1 + fence = next((item for item in fences if item.start_index > index), None) + if fence is None: + bindings.setdefault(ident, []).append( + Binding(marker_line, problem="marker is not followed by a code fence") + ) + continue + + gap = [text_line for text_line in lines[index + 1 : fence.start_index] if text_line.strip()] + if gap: + bindings.setdefault(ident, []).append( + Binding( + marker_line, + problem="non-blank content between marker (line {}) and fence (line {})".format( + marker_line, fence.start_line + ), + ) + ) + continue + + info_tokens = fence.info.split() + if not info_tokens or info_tokens[0].lower() != "sql": + bindings.setdefault(ident, []).append( + Binding( + marker_line, + problem="marker is not followed by a ```sql fence (fence at line {})".format( + fence.start_line + ), + ) + ) + continue + + body = [item for item in fence.lines if item.text.strip()] + if len(body) != 1: + bindings.setdefault(ident, []).append( + Binding( + marker_line, + problem="expected exactly one SQL line in the fenced block at line {}, found {}".format( + fence.start_line, len(body) + ), + ) + ) + continue + + if not fence.closed: + issues.append( + "fenced block at line {} is never closed".format(fence.start_line) + ) + + bindings.setdefault(ident, []).append( + Binding(marker_line, body[0].text.strip(), body[0].number) + ) + + return bindings, issues + + +# -------------------------------------------------------------------------- +# Comparison +# -------------------------------------------------------------------------- + + +def first_difference(left: str, right: str) -> int: + """Return the 0-based index of the first differing character.""" + for position, (a, b) in enumerate(zip(left, right)): + if a != b: + return position + return min(len(left), len(right)) + + +def preview(value: str) -> str: + if len(value) <= PREVIEW_LIMIT: + return value + return value[: PREVIEW_LIMIT - 3] + "..." + + +def evaluate( + manifest: Manifest, + source_text: str, + guide_text: str, +) -> Tuple[List[Result], List[str]]: + statements = extract_source_statements(source_text, manifest.allowed_prompts) + bindings, issues = bind_guide(guide_text, manifest.guide_marker) + results: List[Result] = [] + + known_ids = {example.id for example in manifest.examples} + for ident, found in sorted(bindings.items()): + if ident not in known_ids: + for item in found: + issues.append( + "guide line {}: marker id {!r} is not declared in the manifest".format( + item.line, ident + ) + ) + + for example in manifest.examples: + bound = bindings.get(example.id, []) + + if not bound: + results.append( + Result( + example.id, + False, + "guide marker not found; expected ".format( + manifest.guide_marker, example.id + ), + ) + ) + continue + + if len(bound) > 1: + lines = ", ".join(str(item.line) for item in bound) + results.append( + Result( + example.id, + False, + "guide marker appears {} times (lines {}); each id must appear once".format( + len(bound), lines + ), + ) + ) + continue + + single = bound[0] + if single.problem or single.sql is None: + results.append( + Result(example.id, False, single.problem, guide_line=single.line) + ) + continue + + candidates = [ + item for item in statements if item.sql.startswith(example.statement_prefix) + ] + + if not candidates: + results.append( + Result( + example.id, + False, + "source statement not found; no prompted line starts with {!r}".format( + example.statement_prefix + ), + guide_line=single.sql_line, + actual=single.sql, + ) + ) + continue + + if len(candidates) > 1: + lines = ", ".join(str(item.line) for item in candidates) + results.append( + Result( + example.id, + False, + "source statement is ambiguous: {} candidates with prefix {!r} at lines {}; " + "refusing to pick one".format( + len(candidates), example.statement_prefix, lines + ), + guide_line=single.sql_line, + actual=single.sql, + ) + ) + continue + + source = candidates[0] + if source.sql == single.sql: + results.append( + Result( + example.id, + True, + source_line=source.line, + guide_line=single.sql_line, + ) + ) + continue + + offset = first_difference(source.sql, single.sql) + results.append( + Result( + example.id, + False, + "statement differs from source at column {} (source line {})".format( + offset + 1, source.line + ), + source_line=source.line, + guide_line=single.sql_line, + expected=source.sql, + actual=single.sql, + ) + ) + + return results, issues + + +# -------------------------------------------------------------------------- +# Rendering +# -------------------------------------------------------------------------- + + +def render_text(report: Report) -> str: + lines = [ + "sql-guide source consistency check", + " manifest: {}".format(report.manifest_path), + " source: {}".format(report.source_path), + " guide: {}".format(report.guide_path), + "", + ] + + for result in report.results: + if result.passed: + lines.append( + "PASS {:<22} guide:{} <- source:{}".format( + result.example_id, result.guide_line, result.source_line + ) + ) + continue + + lines.append("FAIL {}".format(result.example_id)) + lines.append(" reason: {}".format(result.reason)) + if result.guide_line is not None: + lines.append(" guide: {}:{}".format(report.guide_path, result.guide_line)) + if result.source_line is not None: + lines.append(" source: {}:{}".format(report.source_path, result.source_line)) + if result.expected is not None: + lines.append(" expected: {}".format(result.expected)) + if result.actual is not None: + lines.append(" actual: {}".format(result.actual)) + + if report.issues: + lines.append("") + lines.append("scope issues:") + for issue in report.issues: + lines.append(" - {}".format(issue)) + + lines.append("") + lines.append( + "summary: {} examples, {} passed, {} failed".format( + report.total, report.passed, report.failed + ) + ) + return "\n".join(lines) + + +def render_json(report: Report) -> str: + payload = { + "schema_version": SCHEMA_VERSION, + "summary": { + "total": report.total, + "passed": report.passed, + "failed": report.failed, + "status": "pass" if report.total and not report.failed and not report.issues else "fail", + }, + "files": { + "manifest": report.manifest_path, + "source": report.source_path, + "guide": report.guide_path, + }, + "results": [ + { + "id": result.example_id, + "status": "pass" if result.passed else "fail", + "reason": result.reason, + "source": {"path": report.source_path, "line": result.source_line}, + "guide": {"path": report.guide_path, "line": result.guide_line}, + "expected": preview(result.expected) if result.expected is not None else None, + "actual": preview(result.actual) if result.actual is not None else None, + } + for result in report.results + ], + "issues": list(report.issues), + } + return json.dumps(payload, ensure_ascii=False, indent=2) + + +# -------------------------------------------------------------------------- +# Entry point +# -------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="check.py", + description=( + "Compare the SQL statements quoted in the companion guide with the " + "statements in the source document. Offline, read-only, never executes SQL." + ), + ) + parser.add_argument( + "--root", + default=".", + help="repository root that contains the manifest and both documents (default: .)", + ) + parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + dest="output_format", + help="report format (default: text)", + ) + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + root = Path(args.root) + if not root.is_dir(): + print("error: --root is not a directory: {}".format(root), file=sys.stderr) + return EXIT_USAGE + + manifest_path = (root / DEFAULT_MANIFEST).resolve() + + try: + manifest = load_manifest(manifest_path) + except ManifestError as exc: + print("error: {}".format(exc), file=sys.stderr) + return EXIT_USAGE + + source_path = (root / manifest.source_document).resolve() + guide_path = (root / manifest.guide_document).resolve() + + source_text = "" + guide_text = "" + missing: List[str] = [] + for label, path in (("source", source_path), ("guide", guide_path)): + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + missing.append("{} document not found: {}".format(label, manifest.source_document if label == "source" else manifest.guide_document)) + continue + except OSError as exc: + missing.append( + "{} document unreadable: {} ({})".format( + label, path, exc + ) + ) + continue + if label == "source": + source_text = text + else: + guide_text = text + + report = Report( + manifest_path=str(DEFAULT_MANIFEST), + source_path=manifest.source_document, + guide_path=manifest.guide_document, + ) + + if missing: + for example in manifest.examples: + report.results.append( + Result(example.id, False, "; ".join(missing)) + ) + else: + results, issues = evaluate(manifest, source_text, guide_text) + report.results = results + report.issues = issues + + if args.output_format == "json": + print(render_json(report)) + else: + print(render_text(report)) + + return report.exit_code() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/sql_guide/examples.json b/tools/sql_guide/examples.json new file mode 100644 index 0000000..52b2991 --- /dev/null +++ b/tools/sql_guide/examples.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "source_document": "docs/guide/01-quickstart.md", + "guide_document": "docs/guide/first-three-sql.md", + "allowed_prompts": [ + "postgres=#", + "test=>" + ], + "guide_marker": "sql-guide-example", + "examples": [ + { + "id": "create-foo-table", + "statement_prefix": "create table foo" + }, + { + "id": "insert-foo-rows", + "statement_prefix": "insert into foo values" + }, + { + "id": "select-foo-all", + "statement_prefix": "select * from foo" + } + ] +} diff --git a/tools/sql_guide/tests/test_check.py b/tools/sql_guide/tests/test_check.py new file mode 100644 index 0000000..ef9552b --- /dev/null +++ b/tools/sql_guide/tests/test_check.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Tests for tools/sql_guide/check.py. + +Every case builds a tiny throw-away repository in a temporary directory, so the +tests never touch the real documents and never need a database or a network +connection. The last test class runs the checker against the real repository. + +Run with: + python3 -m unittest discover -s tools/sql_guide/tests -v +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +CHECK = Path(__file__).resolve().parents[1] / "check.py" +REPO_ROOT = CHECK.parents[2] + +CREATE_SQL = "create table foo(id bigint, str text) distribute by shard(id);" +INSERT_SQL = "insert into foo values(1, 'tencent'), (2, 'shenzhen');" +SELECT_SQL = "select * from foo;" + +# --------------------------------------------------------------------------- +# Fixtures modelled on docs/guide/01-quickstart.md and the companion guide. +# --------------------------------------------------------------------------- + +SOURCE_HEADER = """# 快速入门 + +## 使用 +* 连接到 CN 主节点执行 SQL + +``` +""" + +SOURCE_BODY = """$ psql -h ${{CoordinateNode_IP}} -p ${{CoordinateNode_PORT}} -U opentenbase -d postgres + +postgres=# create database test; +CREATE DATABASE +postgres=# \\c test test +You are now connected to database "test" as user "test". +test=> {create} +CREATE TABLE +test=> {insert} +COPY 2 +test=> {select} + id | str +----+---------- + 1 | tencent + 2 | shenzhen +(2 rows) + +``` +""" + +SOURCE_FIXTURE = SOURCE_HEADER + SOURCE_BODY.format( + create=CREATE_SQL, insert=INSERT_SQL, select=SELECT_SQL +) + + +def guide_block(example_id, sql): + return "\n```sql\n{}\n```\n".format(example_id, sql) + + +GUIDE_HEADER = "# 读懂第一条 SQL:建表、插入与查询\n\n正文。\n\n" + +GUIDE_FIXTURE = ( + GUIDE_HEADER + + guide_block("create-foo-table", CREATE_SQL) + + "\n" + + guide_block("insert-foo-rows", INSERT_SQL) + + "\n" + + guide_block("select-foo-all", SELECT_SQL) +) + +MANIFEST_FIXTURE = { + "schema_version": 1, + "source_document": "docs/guide/01-quickstart.md", + "guide_document": "docs/guide/first-three-sql.md", + "allowed_prompts": ["postgres=#", "test=>"], + "guide_marker": "sql-guide-example", + "examples": [ + {"id": "create-foo-table", "statement_prefix": "create table foo"}, + {"id": "insert-foo-rows", "statement_prefix": "insert into foo values"}, + {"id": "select-foo-all", "statement_prefix": "select * from foo"}, + ], +} + + +class CheckerTestCase(unittest.TestCase): + """Base class: builds a fixture repository and runs the checker on it.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + (self.root / "tools" / "sql_guide").mkdir(parents=True) + (self.root / "docs" / "guide").mkdir(parents=True) + self.manifest_path = self.root / "tools" / "sql_guide" / "examples.json" + self.source_path = self.root / "docs" / "guide" / "01-quickstart.md" + self.guide_path = self.root / "docs" / "guide" / "first-three-sql.md" + self.write_manifest() + self.write_source(SOURCE_FIXTURE) + self.write_guide(GUIDE_FIXTURE) + + def tearDown(self): + self._tmp.cleanup() + + # -- fixture helpers --------------------------------------------------- + + def write_manifest(self, payload=None, raw=None): + if raw is not None: + self.manifest_path.write_text(raw, encoding="utf-8") + return + data = MANIFEST_FIXTURE if payload is None else payload + self.manifest_path.write_text( + json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + def write_source(self, text): + self.source_path.write_text(text, encoding="utf-8") + + def write_guide(self, text): + self.guide_path.write_text(text, encoding="utf-8") + + def line_of(self, path, needle): + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if needle in line: + return number + raise AssertionError("fixture does not contain {!r}".format(needle)) + + # -- runner ------------------------------------------------------------ + + def run_check(self, *extra): + completed = subprocess.run( + [sys.executable, str(CHECK), "--root", str(self.root), *extra], + capture_output=True, + text=True, + encoding="utf-8", + ) + return completed.returncode, completed.stdout, completed.stderr + + +class TestHappyPath(CheckerTestCase): + """T01: the three correct examples pass with exit code 0.""" + + def test_t01_all_examples_pass(self): + code, out, err = self.run_check() + self.assertEqual(code, 0, msg=err) + self.assertIn("summary: 3 examples, 3 passed, 0 failed", out) + self.assertEqual(out.count("PASS "), 3) + self.assertNotIn("FAIL", out) + + def test_t01_reports_source_line_numbers(self): + _, out, _ = self.run_check() + self.assertIn("guide:{} <- source:{}".format( + self.line_of(self.guide_path, CREATE_SQL), + self.line_of(self.source_path, CREATE_SQL), + ), out) + + +class TestMismatch(CheckerTestCase): + """T02 and T08: any byte-level drift is a failure with a precise location.""" + + def test_t02_changed_string_literal_fails(self): + changed = INSERT_SQL.replace("'shenzhen'", "'Shanghai'") + self.write_guide(GUIDE_FIXTURE.replace(INSERT_SQL, changed)) + code, out, err = self.run_check() + self.assertEqual(code, 1, msg=err) + self.assertIn("FAIL insert-foo-rows", out) + self.assertIn("differs from source", out) + self.assertIn("summary: 3 examples, 2 passed, 1 failed", out) + + def test_t02_failure_points_at_the_edited_line(self): + changed = INSERT_SQL.replace("'shenzhen'", "'Shanghai'") + self.write_guide(GUIDE_FIXTURE.replace(INSERT_SQL, changed)) + _, out, _ = self.run_check() + expected_line = self.line_of(self.guide_path, changed) + self.assertIn("guide: docs/guide/first-three-sql.md:{}".format(expected_line), out) + + def test_t08_internal_space_is_not_normalised(self): + spaced = INSERT_SQL.replace("'shenzhen'", "'shen zhen'") + self.write_guide(GUIDE_FIXTURE.replace(INSERT_SQL, spaced)) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("differs from source", out) + + def test_t08_trailing_semicolon_is_compared(self): + self.write_guide(GUIDE_FIXTURE.replace(SELECT_SQL, "select * from foo")) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("FAIL select-foo-all", out) + + +class TestSourceLocating(CheckerTestCase): + """T03, T04, T05, T07: how the source document is located.""" + + def test_t03_missing_source_document(self): + self.source_path.unlink() + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("source document not found: docs/guide/01-quickstart.md", out) + self.assertIn("summary: 3 examples, 0 passed, 3 failed", out) + + def test_t04_source_statement_missing(self): + self.write_source(SOURCE_FIXTURE.replace("test=> " + INSERT_SQL + "\n", "")) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("source statement not found", out) + self.assertIn("FAIL insert-foo-rows", out) + + def test_t05_two_matching_statements_are_ambiguous(self): + duplicated = SOURCE_FIXTURE.replace( + "test=> " + INSERT_SQL + "\n", + "test=> " + INSERT_SQL + "\ntest=> " + INSERT_SQL + "\n", + ) + self.write_source(duplicated) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("ambiguous", out) + self.assertIn("refusing to pick one", out) + + def test_t07_output_lines_are_not_treated_as_statements(self): + # The fixture already mixes prompts, meta-commands and output; a clean + # pass proves that only prompted lines were read, and that the prefix + # match did not pick up "CREATE TABLE" or "COPY 2". + code, out, _ = self.run_check() + self.assertEqual(code, 0) + self.assertNotIn("COPY 2", out) + + def test_t07_statement_outside_a_fence_is_ignored(self): + injected = SOURCE_FIXTURE + "\ntest=> select * from foo;\n" + self.write_source(injected) + code, _, _ = self.run_check() + self.assertEqual(code, 0, msg="a prompt line outside a fence must not be scanned") + + +class TestGuideBinding(CheckerTestCase): + """T06: the guide side of the binding must be unique and well formed.""" + + def test_t06_missing_marker(self): + self.write_guide( + GUIDE_FIXTURE.replace(guide_block("insert-foo-rows", INSERT_SQL), "") + ) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("guide marker not found", out) + self.assertIn("FAIL insert-foo-rows", out) + + def test_t06_duplicate_marker(self): + block = guide_block("create-foo-table", CREATE_SQL) + self.write_guide(GUIDE_FIXTURE + "\n" + block) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("appears 2 times", out) + + def test_t06_marker_without_fence(self): + self.write_guide( + GUIDE_FIXTURE.replace( + guide_block("select-foo-all", SELECT_SQL), + "\n", + ) + ) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("not followed by", out) + + def test_t06_multi_line_block_is_rejected(self): + block = "\n```sql\n{}\n{}\n```\n".format( + CREATE_SQL, SELECT_SQL + ) + self.write_guide( + GUIDE_FIXTURE.replace(guide_block("create-foo-table", CREATE_SQL), block) + ) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("expected exactly one SQL line", out) + + def test_t06_unknown_marker_id_is_reported(self): + self.write_guide( + GUIDE_FIXTURE + "\n" + guide_block("not-in-manifest", SELECT_SQL) + ) + code, out, _ = self.run_check() + self.assertEqual(code, 1) + self.assertIn("is not declared in the manifest", out) + + +class TestManifestValidation(CheckerTestCase): + """T09: a broken manifest is a usage error, never a silent pass.""" + + def test_t09_invalid_json(self): + self.write_manifest(raw="{ not json ") + code, out, err = self.run_check() + self.assertEqual(code, 2) + self.assertIn("not valid JSON", err) + self.assertEqual(out, "") + + def test_t09_missing_key(self): + payload = dict(MANIFEST_FIXTURE) + payload.pop("guide_marker") + self.write_manifest(payload) + code, _, err = self.run_check() + self.assertEqual(code, 2) + self.assertIn("missing key", err) + + def test_t09_unknown_key(self): + payload = json.loads(json.dumps(MANIFEST_FIXTURE)) + payload["unexpected"] = True + self.write_manifest(payload) + code, _, err = self.run_check() + self.assertEqual(code, 2) + self.assertIn("unsupported key", err) + + def test_t09_unsupported_schema_version(self): + payload = json.loads(json.dumps(MANIFEST_FIXTURE)) + payload["schema_version"] = 99 + self.write_manifest(payload) + code, _, err = self.run_check() + self.assertEqual(code, 2) + self.assertIn("unsupported schema_version", err) + + def test_t09_missing_manifest(self): + self.manifest_path.unlink() + code, _, err = self.run_check() + self.assertEqual(code, 2) + self.assertIn("manifest not found", err) + + def test_t09_bad_root(self): + completed = subprocess.run( + [sys.executable, str(CHECK), "--root", str(self.root / "absent")], + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("not a directory", completed.stderr) + + def test_t09_bad_cli_argument(self): + completed = subprocess.run( + [sys.executable, str(CHECK), "--format", "yaml"], + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertEqual(completed.returncode, 2) + + +class TestJsonReport(CheckerTestCase): + """T10: the JSON report is machine readable and agrees with the text one.""" + + def test_t10_json_report_is_consistent(self): + code, out, _ = self.run_check("--format", "json") + self.assertEqual(code, 0) + payload = json.loads(out) + self.assertEqual(payload["schema_version"], 1) + self.assertEqual(payload["summary"]["total"], 3) + self.assertEqual(payload["summary"]["passed"], 3) + self.assertEqual(payload["summary"]["failed"], 0) + self.assertEqual(payload["summary"]["status"], "pass") + self.assertEqual(len(payload["results"]), 3) + self.assertEqual(payload["issues"], []) + for item in payload["results"]: + self.assertEqual(item["status"], "pass") + self.assertIsInstance(item["guide"]["line"], int) + self.assertIsInstance(item["source"]["line"], int) + + def test_t10_json_reports_failures_without_dumping_files(self): + self.write_guide(GUIDE_FIXTURE.replace(INSERT_SQL, INSERT_SQL.replace("tencent", "tencnet"))) + code, out, _ = self.run_check("--format", "json") + self.assertEqual(code, 1) + payload = json.loads(out) + self.assertEqual(payload["summary"]["failed"], 1) + failed = [item for item in payload["results"] if item["status"] == "fail"] + self.assertEqual(len(failed), 1) + self.assertIn("differs from source", failed[0]["reason"]) + + def test_t10_help_exits_zero(self): + completed = subprocess.run( + [sys.executable, str(CHECK), "--help"], + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertEqual(completed.returncode, 0) + self.assertIn("--format", completed.stdout) + + +class TestRealRepository(unittest.TestCase): + """The documents shipped in this repository must satisfy the checker.""" + + def test_real_repository_passes(self): + completed = subprocess.run( + [sys.executable, str(CHECK), "--root", str(REPO_ROOT)], + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertEqual(completed.returncode, 0, msg=completed.stdout + completed.stderr) + self.assertIn("3 examples, 3 passed, 0 failed", completed.stdout) + + def test_real_repository_json_passes(self): + completed = subprocess.run( + [sys.executable, str(CHECK), "--root", str(REPO_ROOT), "--format", "json"], + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertEqual(completed.returncode, 0, msg=completed.stdout + completed.stderr) + self.assertEqual(json.loads(completed.stdout)["summary"]["status"], "pass") + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 38f9786960804cc7939efc994941c7bf8a387f97 Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:42:39 +0800 Subject: [PATCH 2/6] docs: add a flow diagram to the first SQL walkthrough The three statements, their order and the CN/DN path were only described in prose and a table. Add one diagram showing the reading order and the execution path. The diagram deliberately does not place individual rows on individual DNs: distribute by shard(id) only defines how a row is located from its id, and the actual placement depends on the distribution rule and the number of nodes. A caption repeats that limit so the drawing cannot be read as a claim about data placement. SVG is used so the figure stays reviewable as text (6.8 KB, no binary blob) and renders in the existing mkdocs build without new extensions. --- docs/guide/first-three-sql.md | 4 ++ docs/guide/images/first-three-sql-flow.svg | 69 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 docs/guide/images/first-three-sql-flow.svg diff --git a/docs/guide/first-three-sql.md b/docs/guide/first-three-sql.md index 408d2ac..7f3a511 100644 --- a/docs/guide/first-three-sql.md +++ b/docs/guide/first-three-sql.md @@ -148,6 +148,10 @@ select * from foo; ## 把三条连起来看 +![三条 SQL 语句的阅读顺序与执行路径](images/first-three-sql-flow.svg) + +图中只表示执行路径和阅读顺序,不表示数据分布的具体落点:`distribute by shard(id)` 规定“按 `id` 计算落点”,具体落到哪个 DN 由分片规则和节点数量决定,本文不对某一行落在哪个节点下结论。 + | 顺序 | 语句作用 | 依赖 | 来源文档展示的输出 | |---|---|---|---| | 第一步 | 建表 `foo`,两列,按 `id` 分布 | 已建库、建用户、切到 `test` 库 | `CREATE TABLE` | diff --git a/docs/guide/images/first-three-sql-flow.svg b/docs/guide/images/first-three-sql-flow.svg new file mode 100644 index 0000000..34a3d43 --- /dev/null +++ b/docs/guide/images/first-three-sql-flow.svg @@ -0,0 +1,69 @@ + + 三条 SQL 语句的阅读顺序与执行路径 + 客户端通过 psql 连接 CN 协调节点,CN 把语句下发到多个 DN 数据节点。三条语句按建表、插入、查询的顺序执行,每一步都依赖上一步成功。 + + + 三条语句:阅读顺序与执行路径 + 先建表,再插入,最后查询;后一步依赖前一步成功。原文把这三条语句和执行输出放在同一个代码块里。 + + + + psql 客户端 + 提示符 test=> + + + + + + CN 协调节点 + SQL 从这里下发 + + + + + + DN 数据节点(多个) + 数据按分片存放 + + + distribute by shard(id) 只规定 + “按 id 计算落点”,不规定具体 + 落在哪个 DN。 + + + + + 1 + 建表 + CREATE TABLE foo( + id, str) …shard(id) + 来源展示:CREATE TABLE + + + + 依赖 + + + + 2 + 插入两行 + INSERT INTO foo + VALUES (1,…), (2,…) + 来源展示:COPY 2 + + + + 依赖 + + + + 3 + 查询 + SELECT * FROM foo + 无 ORDER BY ⇒ 顺序不保证 + 预期结果:两行 + + 说明:本图只表示执行路径与阅读顺序,不表示数据分布的具体落点;三条语句均未在本文中实际执行。 + 可运行 python3 tools/sql_guide/check.py --root . 检查本文引用的三条语句是否仍与来源文档逐字一致。 + 图 1 三条语句的阅读顺序与执行路径 + From 570a2a0b5a8277abce36549a74585eab449b529f Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:47:15 +0800 Subject: [PATCH 3/6] docs: explain the flow diagram instead of only qualifying it The paragraph after the figure only said what the drawing does not claim, which left the reader to work the figure out alone. Replace it with what the drawing actually shows: the client/CN/DN path on top, the three-step chain below, and the reason no individual row is drawn onto a node. Rename the closing scope section to plain language and drop the disclaimer phrasing. --- docs/guide/first-three-sql.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/guide/first-three-sql.md b/docs/guide/first-three-sql.md index 7f3a511..a70289e 100644 --- a/docs/guide/first-three-sql.md +++ b/docs/guide/first-three-sql.md @@ -150,7 +150,9 @@ select * from foo; ![三条 SQL 语句的阅读顺序与执行路径](images/first-three-sql-flow.svg) -图中只表示执行路径和阅读顺序,不表示数据分布的具体落点:`distribute by shard(id)` 规定“按 `id` 计算落点”,具体落到哪个 DN 由分片规则和节点数量决定,本文不对某一行落在哪个节点下结论。 +先看图的上半部分:客户端连到 CN,CN 再把语句下发到各个 DN,返回的数据由 CN 汇总给客户端。再看下半部分:三步是一条链,建表拿到表结构,插入才有地方放数据,查询才有内容可读;任何一步失败,后面两步都做不下去。 + +图中的 `distribute by shard(id)` 只规定“由 `id` 计算落点”,具体落在哪个 DN 取决于分片规则和节点数量,所以图上没有把某一行画到某个具体节点。 | 顺序 | 语句作用 | 依赖 | 来源文档展示的输出 | |---|---|---|---| @@ -182,9 +184,9 @@ python3 tools/sql_guide/check.py --root . 4. [用户管理](07-user-rights.md):理解本文示例里的 `test` 用户、库属主和权限之间的关系。 5. [如何贡献 OpenTenBase](../contribution/how-to.md):发现文档问题或有改进建议时,提 issue 或 PR 的流程。 -## 本文的边界 +## 这一页覆盖到哪里 -- 三条语句逐字取自来源文档,本文不改写,也不新增第四条。 -- 本文未在真实环境执行这些语句;标注为“来源文档展示”的输出是原文照录,标注为“预期结果”的内容由原文展示推导而来。 -- 本文不涉及性能调优、分布键的选择策略、权限模型的细节。 -- 引用一致性可以借助上述工具复查;解释是否准确仍需人工评审。如果发现解释与你的实际行为不一致,欢迎按第 5 条入口反馈。 +- 只讲上面这三条语句,不新增第四条,也不改写原文。 +- 输出部分是原文照录(标为“来源文档展示”)或由原文推导(标为“预期结果”),没有在集群上实跑。 +- 性能调优、分布键怎么选、权限模型的细节,留给后面的章节。 +- 如果你的环境跑出来和这里写的不一样,欢迎按上面第 5 个入口反馈。 From 8f877398b934928087972ce640e3b36bcd038618 Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:48:59 +0800 Subject: [PATCH 4/6] docs: drop the repeated provenance disclaimers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The label on each statement already says the output is quoted from the source document; repeating "(原文照录,本文未执行)" after every one of them only adds noise. State the rule once at the top and drop the repetitions. --- docs/guide/first-three-sql.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/guide/first-three-sql.md b/docs/guide/first-three-sql.md index a70289e..032fd44 100644 --- a/docs/guide/first-three-sql.md +++ b/docs/guide/first-three-sql.md @@ -2,7 +2,7 @@ > 本文伴读[快速入门](01-quickstart.md)的“使用”小节。那里给出了三条 SQL 和它们的执行输出,但没有逐条说明每条语句在做什么。本文按“建表 → 插入 → 查询”的顺序把这三条拆开讲清楚。 > -> 本文不新增 SQL,也不改写原文中的 SQL。文中标注为“来源文档展示”的输出,均为原文照录,不是本文作者的实测结果。 +> 本文不新增 SQL,也不改写原文中的 SQL。标为“来源文档展示”的输出是原文照录,不是实测结果。 ## 先分清三种提示符 @@ -56,7 +56,7 @@ create table foo(id bigint, str text) distribute by shard(id); **前置条件**:已按上文完成建库、建用户并切换到 `test` 库。 -**来源文档展示**:紧跟这条语句,原文给出的标签是 `CREATE TABLE`。(原文照录,本文未执行。) +**来源文档展示**:紧跟这条语句,原文给出的标签是 `CREATE TABLE`。 **常见误解**: @@ -90,7 +90,7 @@ insert into foo values(1, 'tencent'), (2, 'shenzhen'); **前置条件**:`foo` 表已存在,即第一步已成功。否则插入会失败。 -**来源文档展示**:紧跟这条语句,原文给出的标签是 `COPY 2`。(原文照录。) +**来源文档展示**:紧跟这条语句,原文给出的标签是 `COPY 2`。 > 说明:本文只记录原文写了什么,不对这个标签与当前版本的实际输出是否一致作判断。如果你在自己的环境里执行,请以实际输出为准。 @@ -136,7 +136,7 @@ select * from foo; **前置条件**:`foo` 表已存在且有数据,即前两步都已成功。 -**来源文档展示**:原文在结果表格后给出 `(2 rows)`,表示返回两行。(原文照录。) +**来源文档展示**:原文在结果表格后给出 `(2 rows)`,表示返回两行。 **常见误解**: @@ -187,6 +187,6 @@ python3 tools/sql_guide/check.py --root . ## 这一页覆盖到哪里 - 只讲上面这三条语句,不新增第四条,也不改写原文。 -- 输出部分是原文照录(标为“来源文档展示”)或由原文推导(标为“预期结果”),没有在集群上实跑。 +- 输出只有两种来源:原文照录,或由原文推导的预期结果;没有在集群上实跑。 - 性能调优、分布键怎么选、权限模型的细节,留给后面的章节。 - 如果你的环境跑出来和这里写的不一样,欢迎按上面第 5 个入口反馈。 From d4179a9dbab4317a74d12c1218f585317e91d48b Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:49:23 +0800 Subject: [PATCH 5/6] docs: state the COPY 2 note plainly instead of describing what it will not judge --- docs/guide/first-three-sql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/first-three-sql.md b/docs/guide/first-three-sql.md index 032fd44..ac942d8 100644 --- a/docs/guide/first-three-sql.md +++ b/docs/guide/first-three-sql.md @@ -92,7 +92,7 @@ insert into foo values(1, 'tencent'), (2, 'shenzhen'); **来源文档展示**:紧跟这条语句,原文给出的标签是 `COPY 2`。 -> 说明:本文只记录原文写了什么,不对这个标签与当前版本的实际输出是否一致作判断。如果你在自己的环境里执行,请以实际输出为准。 +> 说明:`COPY 2` 是原文给出的标签,这里照录。如果你在自己的环境里执行,以实际输出为准。 **常见误解**: From c63030350091892c83c5ef9c67af8e5c186772d5 Mon Sep 17 00:00:00 2001 From: shibuchou <134493576+shibuchou@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:57:12 +0800 Subject: [PATCH 6/6] docs: drop the numbered caption baked into the flow diagram; the figure caption belongs to the document, not the image --- docs/guide/images/first-three-sql-flow.svg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/guide/images/first-three-sql-flow.svg b/docs/guide/images/first-three-sql-flow.svg index 34a3d43..a756c24 100644 --- a/docs/guide/images/first-three-sql-flow.svg +++ b/docs/guide/images/first-three-sql-flow.svg @@ -1,8 +1,8 @@ - + 三条 SQL 语句的阅读顺序与执行路径 客户端通过 psql 连接 CN 协调节点,CN 把语句下发到多个 DN 数据节点。三条语句按建表、插入、查询的顺序执行,每一步都依赖上一步成功。 - + 三条语句:阅读顺序与执行路径 先建表,再插入,最后查询;后一步依赖前一步成功。原文把这三条语句和执行输出放在同一个代码块里。 @@ -65,5 +65,4 @@ 说明:本图只表示执行路径与阅读顺序,不表示数据分布的具体落点;三条语句均未在本文中实际执行。 可运行 python3 tools/sql_guide/check.py --root . 检查本文引用的三条语句是否仍与来源文档逐字一致。 - 图 1 三条语句的阅读顺序与执行路径