diff --git a/packages/bugc/CHANGELOG.md b/packages/bugc/CHANGELOG.md index 456c2f36fa..ed8e7e6200 100644 --- a/packages/bugc/CHANGELOG.md +++ b/packages/bugc/CHANGELOG.md @@ -79,6 +79,11 @@ support. Changes to the specification itself are tracked in the root ### Fixed +- A source range now ends at its node's last token. Before, a node + followed by whitespace or a comment included them, so most `code` + ranges had trailing whitespace: `block.number ` in `block.number + 1`, + the target `x ` in `x = 1`, a statement up to the next statement, and + the `create` and `code` blocks up to the next block ([#362]). - A call to a function with no return type now compiles as a statement, as in `bump();`, at every optimization level. The function's `return` context has no `data`. Before, IR generation failed with "Cannot convert @@ -290,3 +295,4 @@ First publication. [#349]: https://github.com/ethdebug/format/pull/349 [#351]: https://github.com/ethdebug/format/pull/351 [#352]: https://github.com/ethdebug/format/pull/352 +[#362]: https://github.com/ethdebug/format/pull/362 diff --git a/packages/bugc/src/parser/locations.test.ts b/packages/bugc/src/parser/locations.test.ts new file mode 100644 index 0000000000..2978cbd3c9 --- /dev/null +++ b/packages/bugc/src/parser/locations.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect } from "vitest"; +import { promises as fs } from "fs"; +import path from "path"; +import { glob } from "glob"; + +import { parse } from "./parser.js"; + +const EXAMPLES_DIR = path.resolve(__dirname, "../../examples"); + +/** + * The spans of the comments in a source, skipping string literals + */ +function commentSpans(source: string): [number, number][] { + const spans: [number, number][] = []; + let i = 0; + while (i < source.length) { + if (source[i] === '"') { + const close = source.indexOf('"', i + 1); + i = close === -1 ? source.length : close + 1; + } else if (source.startsWith("//", i)) { + const newline = source.indexOf("\n", i); + const end = newline === -1 ? source.length : newline; + spans.push([i, end]); + i = end; + } else if (source.startsWith("/*", i)) { + const close = source.indexOf("*/", i + 2); + const end = close === -1 ? source.length : close + 2; + spans.push([i, end]); + i = end; + } else { + i++; + } + } + return spans; +} + +/** + * Every source location in an AST, with the node's kind + */ +function locations( + node: unknown, + found: { kind: string; offset: number; length: number }[] = [], +) { + if (Array.isArray(node)) { + for (const item of node) { + locations(item, found); + } + } else if (node instanceof Map) { + for (const value of node.values()) { + locations(value, found); + } + } else if (node && typeof node === "object") { + const { kind, loc } = node as { + kind?: string; + loc?: { offset: number; length: number } | null; + }; + if (loc) { + found.push({ + kind: kind ?? "?", + offset: Number(loc.offset), + length: Number(loc.length), + }); + } + for (const value of Object.values(node)) { + locations(value, found); + } + } + return found; +} + +describe("source locations", () => { + it("should end each node at its last token", async () => { + const files = await glob("**/*.bug", { cwd: EXAMPLES_DIR }); + expect(files.length).toBeGreaterThan(0); + + for (const file of files) { + const source = await fs.readFile(path.join(EXAMPLES_DIR, file), "utf-8"); + const result = parse(source); + if (!result.success) { + continue; + } + + const comments = commentSpans(source); + for (const { kind, offset, length } of locations(result.value)) { + const text = source.slice(offset, offset + length); + const end = offset + length; + const where = `${file}: ${kind} ${JSON.stringify(text)}`; + + expect(text.trim(), where).toBe(text); + expect( + comments.some(([start, stop]) => start < end && end <= stop), + where, + ).toBe(false); + } + } + }); +}); diff --git a/packages/bugc/src/parser/parser.ts b/packages/bugc/src/parser/parser.ts index cf6e5aabfd..ad555b862e 100644 --- a/packages/bugc/src/parser/parser.ts +++ b/packages/bugc/src/parser/parser.ts @@ -16,6 +16,7 @@ const PENDING_ID = "" as Ast.Id; * Parse a BUG program and return Result */ export function parse(input: string): Result { + tokenEnds.clear(); return runParser(parser, input); } @@ -29,10 +30,22 @@ export function parse(input: string): Result { function toSourceLocation(mark: P.Mark): Ast.SourceLocation { return { offset: mark.start.offset, - length: mark.end.offset - mark.start.offset, + length: tokenEnd(mark.start.offset, mark.end.offset) - mark.start.offset, }; } +/** + * For the input being parsed: the end of each token, before the + * whitespace and comments after it, by the index after them. A node's + * location ends at its last token's end, not after the whitespace and + * comments that follow it. + */ +const tokenEnds = new Map(); + +function tokenEnd(start: number, end: number): number { + return Math.max(start, tokenEnds.get(end) ?? end); +} + /** * Basic parsers */ @@ -51,7 +64,11 @@ const _ = whitespaceOrComment.many().result(undefined); const __ = whitespaceOrComment.atLeast(1).result(undefined); // Lexeme: consume trailing whitespace/comments -const lexeme = (p: P.Parser): P.Parser => p.skip(_); +const lexeme = (p: P.Parser): P.Parser => + P.seqMap(p, P.index, _, P.index, (value, end, _skipped, next) => { + tokenEnds.set(next.offset, end.offset); + return value; + }); // Keywords - must not be followed by identifier chars const keyword = (name: string): P.Parser => @@ -651,7 +668,7 @@ const postfixExpression = P.lazy(() => { suffixes.reduce((obj, [suffix, end]) => { const loc = { offset: start.offset, - length: end.offset - start.offset, + length: tokenEnd(start.offset, end.offset) - start.offset, }; const id = `${loc.offset}_${loc.length}` as Ast.Id;