From aa55b6f53fc19117094e702a7f09ee2b8f911310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Vogel?= <104342347+cedricvogel-leo@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:44:40 +0100 Subject: [PATCH] Tests: compare values, not their display (P32) An example passes when the expected text and the output are each one Python literal and equal with ==: 4 for 4.0, [1,2,30] for [1, 2, 30], "+" for '+'. Everything else, and an example with a comparison flag of its own, is checked as doctest does. The report keeps both texts. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 13 +++++++- scripts/check-console-browser.mjs | 7 ++-- scripts/check-runtime-doctests.py | 55 +++++++++++++++++++++++++++++++ src/runtime/doctests.py | 27 ++++++++++++++- 5 files changed, 98 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 92ac1c0..fc34b36 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,7 @@ There is no unit test framework. The automated checks are `test:protocol` (plain Worker messages: - `run` → `result`/`error`; while it runs, `output` messages (`{id, text}`, `id` undefined for a program, the input's id for a REPL input) stream what was printed, at most 50 per 50 ms. The final message brings the whole output again. -- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed). With `coverage: true` the report has `uncovered` (`doctests.py` records the lines with `sys.monitoring` during the examples only; `check-runtime-coverage.py` compares with Python's `trace`); the UI shows it only when all examples pass. +- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed). Examples compare values: `_GlpythonOutputChecker` passes an example whose expected text and output are each one literal and equal with `==` (`4` for `4.0`), else doctest's own check. With `coverage: true` the report has `uncovered` (`doctests.py` records the lines with `sys.monitoring` during the examples only; `check-runtime-coverage.py` compares with Python's `trace`); the UI shows it only when all examples pass. - `run-visual` → `trace-result` (`{trace, ast, output, graphics, error, errorMessage, truncated}`) - `repl` → `repl-result` - `run-visual-repl` (a doctest example) → `trace-result` with `rootPath`/`sources`, or `repl-result` if the input doesn't parse diff --git a/README.md b/README.md index 0058c26..aed95c1 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,18 @@ examples passed, then each failing example in doctest's own words (`Failed example:`, `Expected:`, `Got:` stay untranslated) with a button that visualizes that example step by step against the same definitions. If the example didn't raise, the visualization's last step shows the two texts doctest compared next -to the final value (`exampleGotExpected`, e.g. «Erhalten: 4.0 · Erwartet: 4»). An example that printed +to the final value (`exampleGotExpected`, e.g. «Erhalten: 4.5 · Erwartet: 4»). + +Unlike Python's doctest, an example compares **values, not their display**: +when the expected text and the output are each one Python literal (numbers, +strings, lists, tuples, dicts, sets, `True`/`False`/`None`, nested), it passes +if the two values are equal with `==` - `4` for `4.0`, `[1,2,30]` for +`[1, 2, 30]`, `["+", 1]` for `['+', 1]`, and, as `==` says, `1` for `True` +(no tolerance: `0.3` for `0.1 + 0.2` fails). Anything else - exceptions, +several lines of output, objects such as ``, other text - +and an example with a comparison flag of its own (`ELLIPSIS`, +`NORMALIZE_WHITESPACE`, …) is compared as doctest does. The report shows both +texts as written. An example that printed what was expected but returned `None` passes, as in doctest, with a separate glpython hint that it probably should `return` instead. Texts: `runTests`, `testsRunning`, `testsSummary`, `testsNone`, `testsLoadError`, diff --git a/scripts/check-console-browser.mjs b/scripts/check-console-browser.mjs index 8d49187..e292f77 100644 --- a/scripts/check-console-browser.mjs +++ b/scripts/check-console-browser.mjs @@ -38,10 +38,11 @@ const harness = ` const program = (examples) => `def schnitt(a, b):\n """\n${examples.map(([call, want]) => ` >>> ${call}\n ${want}\n`).join('')} """\n return (a + b) / 2\n` // Enough failing examples that the report is taller than the console. -const failing = program([['schnitt(4, 6)', '5'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']]) +const failing = program([['schnitt(4, 6)', '6'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']]) const edited = program([['schnitt(4, 6)', '5.0'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']]) -const passing = program([['schnitt(4, 6)', '5.0']]) -const passingEdited = program([['schnitt(4, 6)', '5.0'], ['schnitt(1, 2)', '1.5']]) +// "5" for 5.0 passes: Tests compares values, not their display. +const passing = program([['schnitt(4, 6)', '5']]) +const passingEdited = program([['schnitt(4, 6)', '5'], ['schnitt(1, 2)', '1.5']]) let failed = 0 const browser = await chromium.launch({ channel: 'chrome', headless: true }) diff --git a/scripts/check-runtime-doctests.py b/scripts/check-runtime-doctests.py index a1a45be..5c2cc76 100644 --- a/scripts/check-runtime-doctests.py +++ b/scripts/check-runtime-doctests.py @@ -115,6 +115,61 @@ def viertel(x): ''') expect("import: attempted/failed", (report.get("attempted"), report.get("failed")), (1, 0)) +# Values, not their display: one literal on each side passes when the two +# values are equal with ==. +def example(source, want, body): + return f'def f():\n """\n >>> {source}\n {want}\n """\n {body}\n' + + +def outcome(label, source, want, body, passes): + report, _ = run(example(source, want, body)) + expect(f"{label}: attempted/failed", (report.get("attempted"), report.get("failed")), (1, 0 if passes else 1)) + return report + + +outcome("4 for 4.0", "f()", "4", "return 8 / 2", True) +outcome("5.0 for 5", "f()", "5.0", "return 5", True) +outcome("list spacing", "f()", "[1,2,30]", "return [1, 2, 30]", True) +outcome("quote style", "f()", '"+"', 'return "+"', True) +outcome("nested lists with strings", "f()", '["+", 1, ["*", 2, 3]]', "return ['+', 1, ['*', 2, 3]]", True) +outcome("dict, tuple, set, None", "f()", "({'a':1}, (1,2), {3}, None)", "return ({'a': 1}, (1, 2), {3}, None)", True) +outcome("written over several lines", "f()", "[1,\n 2]", "return [1, 2]", False) +# True == 1 in Python; doctest itself already accepts True for 1. +outcome("True for 1", "f()", "1", "return True", True) +outcome("1 for True", "f()", "True", "return 1", True) +# No tolerance: 0.1 + 0.2 != 0.3. +outcome("float", "f()", "0.3", "return 0.1 + 0.2", False) +# A string is not its content: '4' is not 4. +outcome("string for number", "f()", "4", "return '4'", False) +# A wrong value: the report shows both texts as written. +report = outcome("wrong value", "f()", "[1,2,3]", "return [1, 2, 30]", False) +failed = report.get("failures", [{}])[0] +expect("wrong value: want/got", (failed.get("want"), failed.get("got")), ("[1,2,3]\n", "[1, 2, 30]\n")) +for part in ("Expected:\n [1,2,3]\n", "Got:\n [1, 2, 30]\n"): + if part not in failed.get("text", ""): + failures.append(f"wrong value: report lacks {part!r}: {failed.get('text')!r}") +# Expected exceptions work as before; an unexpected one still fails. +outcome("expected exception", "f()", "Traceback (most recent call last):\n ...\n ZeroDivisionError: division by zero", "return 1 / 0", True) +outcome("other exception", "f()", "Traceback (most recent call last):\n ...\n ValueError: x", "return 1 / 0", False) +# Several lines of output: compared as text. +outcome("several lines", "f()", "1\n 2", "print(1); print(2)", True) +outcome("several lines, other spacing", "f()", "[1,2]\n 3", "print([1, 2]); print(3)", False) +# What print wrote is compared as its text would be: one literal on a line +# counts as that value, so print(5) passes for 5.0 as return 5 does; the +# print hint stays. +report = outcome("print", "f()", "5.0", "print(5)", True) +expect("print: hint", report.get("printHints"), [{"source": "f()", "output": "5"}]) +outcome("print text", "f()", "Hallo", "print('Hallo')", True) +outcome("print text, other", "f()", "hallo", "print('Hallo')", False) +# Not literals: compared as text. +outcome("function object", "f", "", "return 1", False) +outcome("set()", "f()", "set()", "return set()", True) +outcome("set() for {}", "f()", "{}", "return set()", False) +# An example's own comparison flag: doctest's check alone. +outcome("NORMALIZE_WHITESPACE", "f() # doctest: +NORMALIZE_WHITESPACE", "[1,2]", "return [1, 2]", False) +outcome("ELLIPSIS", "f() # doctest: +ELLIPSIS", "[1, ...]", "return [1, 2]", True) +outcome("ELLIPSIS, no value comparison", "f() # doctest: +ELLIPSIS", "4", "return 4.0", False) + # No examples at all. report, _ = run("def f(x):\n return x\n") expect("no examples", (report.get("attempted"), report.get("failed"), report.get("failures"), report.get("printHints")), (0, 0, [], [])) diff --git a/src/runtime/doctests.py b/src/runtime/doctests.py index 3b8dbc4..dc6a115 100644 --- a/src/runtime/doctests.py +++ b/src/runtime/doctests.py @@ -13,13 +13,38 @@ # _glpython_run_visual_repl). _glpython_test_namespace = {} +# Compares values, not their display: when the expected text and the output +# are each one Python literal (numbers, strings, lists, tuples, dicts, sets, +# True/False/None, nested), the example passes if the two values are equal +# with ==, so "4" matches 4.0, "[1,2,30]" matches [1, 2, 30] and "+" in +# double quotes matches '+'. Everything else - exceptions, several lines, +# objects like , other text - and every example with a +# comparison flag of its own (ELLIPSIS, NORMALIZE_WHITESPACE, ...) is checked +# as doctest does. +def _glpython_literal(text): + text = text.removesuffix("\n") + if not text.strip() or "\n" in text: + raise ValueError("not one line") + return ast.literal_eval(text.strip()) + +class _GlpythonOutputChecker(doctest.OutputChecker): + def check_output(self, want, got, optionflags): + if super().check_output(want, got, optionflags): + return True + if optionflags & doctest.COMPARISON_FLAGS: + return False + try: + return bool(_glpython_literal(want) == _glpython_literal(got)) + except Exception: + return False + class _GlpythonDocTestRunner(doctest.DocTestRunner): # Records each failing example with doctest's own report text ("Failed # example:", "Expected:", "Got:" - never translated), and notes examples # that printed what was expected while the call itself returned None: they # pass in doctest, but usually mean print was used instead of return. def __init__(self): - super().__init__(verbose=False) + super().__init__(checker=_GlpythonOutputChecker(), verbose=False) self.failed_examples = [] self.print_hints = [] self._displayed = []