Tests: compare values, not their display (P32)

CLAUDE.md

@@ -56,7 +56,7 @@ There is no unit test framework. The automated checks are `test:protocol` (plain
 
 Worker messages:
 - `run` → `result`/`error`; while it runs, `output` messages (`{id, text}`, `id` undefined for a program, the input's id for a REPL input) stream what was printed, at most 50 per 50 ms. The final message brings the whole output again.
-- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed). With `coverage: true` the report has `uncovered` (`doctests.py` records the lines with `sys.monitoring` during the examples only; `check-runtime-coverage.py` compares with Python's `trace`); the UI shows it only when all examples pass.
+- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed). Examples compare values: `_GlpythonOutputChecker` passes an example whose expected text and output are each one literal and equal with `==` (`4` for `4.0`), else doctest's own check. With `coverage: true` the report has `uncovered` (`doctests.py` records the lines with `sys.monitoring` during the examples only; `check-runtime-coverage.py` compares with Python's `trace`); the UI shows it only when all examples pass.
 - `run-visual` → `trace-result` (`{trace, ast, output, graphics, error, errorMessage, truncated}`)
 - `repl` → `repl-result`
 - `run-visual-repl` (a doctest example) → `trace-result` with `rootPath`/`sources`, or `repl-result` if the input doesn't parse

README.md

@@ -153,7 +153,18 @@ examples passed, then each failing example in doctest's own words (`Failed
 example:`, `Expected:`, `Got:` stay untranslated) with a button that visualizes
 that example step by step against the same definitions. If the example didn't
 raise, the visualization's last step shows the two texts doctest compared next
-to the final value (`exampleGotExpected`, e.g. «Erhalten: 4.0 · Erwartet: 4»). An example that printed
+to the final value (`exampleGotExpected`, e.g. «Erhalten: 4.5 · Erwartet: 4»).
+
+Unlike Python's doctest, an example compares **values, not their display**:
+when the expected text and the output are each one Python literal (numbers,
+strings, lists, tuples, dicts, sets, `True`/`False`/`None`, nested), it passes
+if the two values are equal with `==` - `4` for `4.0`, `[1,2,30]` for
+`[1, 2, 30]`, `["+", 1]` for `['+', 1]`, and, as `==` says, `1` for `True`
+(no tolerance: `0.3` for `0.1 + 0.2` fails). Anything else - exceptions,
+several lines of output, objects such as `<function f at 0x…>`, other text -
+and an example with a comparison flag of its own (`ELLIPSIS`,
+`NORMALIZE_WHITESPACE`, …) is compared as doctest does. The report shows both
+texts as written. An example that printed
 what was expected but returned `None` passes, as in doctest, with a separate
 glpython hint that it probably should `return` instead. Texts: `runTests`,
 `testsRunning`, `testsSummary`, `testsNone`, `testsLoadError`,

scripts/check-console-browser.mjs

@@ -38,10 +38,11 @@ const harness = `<!doctype html><meta charset="utf-8"><body style="margin:0">
 
 const program = (examples) => `def schnitt(a, b):\n    """\n${examples.map(([call, want]) => `    >>> ${call}\n    ${want}\n`).join('')}    """\n    return (a + b) / 2\n`
 // Enough failing examples that the report is taller than the console.
-const failing = program([['schnitt(4, 6)', '5'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']])
+const failing = program([['schnitt(4, 6)', '6'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']])
 const edited = program([['schnitt(4, 6)', '5.0'], ['schnitt(6, 4)', '4.5'], ['schnitt(2, 2)', '3'], ['schnitt(1, 1)', '7'], ['schnitt(0, 4)', '1']])
-const passing = program([['schnitt(4, 6)', '5.0']])
-const passingEdited = program([['schnitt(4, 6)', '5.0'], ['schnitt(1, 2)', '1.5']])
+// "5" for 5.0 passes: Tests compares values, not their display.
+const passing = program([['schnitt(4, 6)', '5']])
+const passingEdited = program([['schnitt(4, 6)', '5'], ['schnitt(1, 2)', '1.5']])
 
 let failed = 0
 const browser = await chromium.launch({ channel: 'chrome', headless: true })

scripts/check-runtime-doctests.py

@@ -115,6 +115,61 @@ def viertel(x):
 ''')
 expect("import: attempted/failed", (report.get("attempted"), report.get("failed")), (1, 0))
 
+# Values, not their display: one literal on each side passes when the two
+# values are equal with ==.
+def example(source, want, body):
+    return f'def f():\n    """\n    >>> {source}\n    {want}\n    """\n    {body}\n'
+
+
+def outcome(label, source, want, body, passes):
+    report, _ = run(example(source, want, body))
+    expect(f"{label}: attempted/failed", (report.get("attempted"), report.get("failed")), (1, 0 if passes else 1))
+    return report
+
+
+outcome("4 for 4.0", "f()", "4", "return 8 / 2", True)
+outcome("5.0 for 5", "f()", "5.0", "return 5", True)
+outcome("list spacing", "f()", "[1,2,30]", "return [1, 2, 30]", True)
+outcome("quote style", "f()", '"+"', 'return "+"', True)
+outcome("nested lists with strings", "f()", '["+", 1, ["*", 2, 3]]', "return ['+', 1, ['*', 2, 3]]", True)
+outcome("dict, tuple, set, None", "f()", "({'a':1}, (1,2), {3}, None)", "return ({'a': 1}, (1, 2), {3}, None)", True)
+outcome("written over several lines", "f()", "[1,\n     2]", "return [1, 2]", False)
+# True == 1 in Python; doctest itself already accepts True for 1.
+outcome("True for 1", "f()", "1", "return True", True)
+outcome("1 for True", "f()", "True", "return 1", True)
+# No tolerance: 0.1 + 0.2 != 0.3.
+outcome("float", "f()", "0.3", "return 0.1 + 0.2", False)
+# A string is not its content: '4' is not 4.
+outcome("string for number", "f()", "4", "return '4'", False)
+# A wrong value: the report shows both texts as written.
+report = outcome("wrong value", "f()", "[1,2,3]", "return [1, 2, 30]", False)
+failed = report.get("failures", [{}])[0]
+expect("wrong value: want/got", (failed.get("want"), failed.get("got")), ("[1,2,3]\n", "[1, 2, 30]\n"))
+for part in ("Expected:\n    [1,2,3]\n", "Got:\n    [1, 2, 30]\n"):
+    if part not in failed.get("text", ""):
+        failures.append(f"wrong value: report lacks {part!r}: {failed.get('text')!r}")
+# Expected exceptions work as before; an unexpected one still fails.
+outcome("expected exception", "f()", "Traceback (most recent call last):\n    ...\n    ZeroDivisionError: division by zero", "return 1 / 0", True)
+outcome("other exception", "f()", "Traceback (most recent call last):\n    ...\n    ValueError: x", "return 1 / 0", False)
+# Several lines of output: compared as text.
+outcome("several lines", "f()", "1\n    2", "print(1); print(2)", True)
+outcome("several lines, other spacing", "f()", "[1,2]\n    3", "print([1, 2]); print(3)", False)
+# What print wrote is compared as its text would be: one literal on a line
+# counts as that value, so print(5) passes for 5.0 as return 5 does; the
+# print hint stays.
+report = outcome("print", "f()", "5.0", "print(5)", True)
+expect("print: hint", report.get("printHints"), [{"source": "f()", "output": "5"}])
+outcome("print text", "f()", "Hallo", "print('Hallo')", True)
+outcome("print text, other", "f()", "hallo", "print('Hallo')", False)
+# Not literals: compared as text.
+outcome("function object", "f", "<function f at 0x0>", "return 1", False)
+outcome("set()", "f()", "set()", "return set()", True)
+outcome("set() for {}", "f()", "{}", "return set()", False)
+# An example's own comparison flag: doctest's check alone.
+outcome("NORMALIZE_WHITESPACE", "f()  # doctest: +NORMALIZE_WHITESPACE", "[1,2]", "return [1, 2]", False)
+outcome("ELLIPSIS", "f()  # doctest: +ELLIPSIS", "[1, ...]", "return [1, 2]", True)
+outcome("ELLIPSIS, no value comparison", "f()  # doctest: +ELLIPSIS", "4", "return 4.0", False)
+
 # No examples at all.
 report, _ = run("def f(x):\n    return x\n")
 expect("no examples", (report.get("attempted"), report.get("failed"), report.get("failures"), report.get("printHints")), (0, 0, [], []))

src/runtime/doctests.py

@@ -13,13 +13,38 @@
 # _glpython_run_visual_repl).
 _glpython_test_namespace = {}
 
+# Compares values, not their display: when the expected text and the output
+# are each one Python literal (numbers, strings, lists, tuples, dicts, sets,
+# True/False/None, nested), the example passes if the two values are equal
+# with ==, so "4" matches 4.0, "[1,2,30]" matches [1, 2, 30] and "+" in
+# double quotes matches '+'. Everything else - exceptions, several lines,
+# objects like <function f at 0x...>, other text - and every example with a
+# comparison flag of its own (ELLIPSIS, NORMALIZE_WHITESPACE, ...) is checked
+# as doctest does.
+def _glpython_literal(text):
+  text = text.removesuffix("\n")
+  if not text.strip() or "\n" in text:
+    raise ValueError("not one line")
+  return ast.literal_eval(text.strip())
+
+class _GlpythonOutputChecker(doctest.OutputChecker):
+  def check_output(self, want, got, optionflags):
+    if super().check_output(want, got, optionflags):
+      return True
+    if optionflags & doctest.COMPARISON_FLAGS:
+      return False
+    try:
+      return bool(_glpython_literal(want) == _glpython_literal(got))
+    except Exception:
+      return False
+
 class _GlpythonDocTestRunner(doctest.DocTestRunner):
   # Records each failing example with doctest's own report text ("Failed
   # example:", "Expected:", "Got:" - never translated), and notes examples
   # that printed what was expected while the call itself returned None: they
   # pass in doctest, but usually mean print was used instead of return.
   def __init__(self):
-    super().__init__(verbose=False)
+    super().__init__(checker=_GlpythonOutputChecker(), verbose=False)
     self.failed_examples = []
     self.print_hints = []
     self._displayed = []