diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a42ed42..9137ca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - run: npm run test:hints # The runtime under CPython: the visualizer's columns, docstrings, # condition comments, readable types and argument checks, the doctest - # runner, and which errors get a hint. + # runner and its coverage, and which errors get a hint. - run: npm run test:runtime # The console in Chrome (on the runner) with Pyodide from its CDN, # driving the embed built above: Tests → edit → Tests shows the new diff --git a/CLAUDE.md b/CLAUDE.md index 6dc1e53..83a0bc1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ npm run build # tsc -b && vite build && copy src/style.css -> dist/ (li npm run dev # vite build --watch, for use with npm link/yalc in a host shell npm run lint # oxlint npm run test:protocol # builds the library, then runs scripts/check-embed-protocol.mjs against dist/ -npm run test:runtime # the runtime under CPython: scripts/check-runtime-{columns,values,doctests,calls,hints,fidelity,replay}.py +npm run test:runtime # the runtime under CPython: scripts/check-runtime-{columns,values,doctests,coverage,calls,hints,fidelity,replay,…}.py npm run test:hints # the error hints' wording (de/en) against the built dist/ (run a build first) npm run test:console # the console in Chrome with Pyodide, against dist-embed/ (run build:embed first) npm run build:embed # builds the iframe app + demo page into dist-embed/ @@ -56,7 +56,7 @@ There is no unit test framework. The automated checks are `test:protocol` (plain Worker messages: - `run` → `result`/`error`; while it runs, `output` messages (`{id, text}`, `id` undefined for a program, the input's id for a REPL input) stream what was printed, at most 50 per 50 ms. The final message brings the whole output again. -- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed) +- `test` → first as `run` (`result`/`error`, a fresh REPL session from these files), then `test-result` (the doctests' report; `{skipped: true}` if the file failed). With `coverage: true` the report has `uncovered` (`doctests.py` records the lines with `sys.monitoring` during the examples only; `check-runtime-coverage.py` compares with Python's `trace`); the UI shows it only when all examples pass. - `run-visual` → `trace-result` (`{trace, ast, output, graphics, error, errorMessage, truncated}`) - `repl` → `repl-result` - `run-visual-repl` (a doctest example) → `trace-result` with `rootPath`/`sources`, or `repl-result` if the input doesn't parse diff --git a/README.md b/README.md index 2c5cd66..9624b59 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,20 @@ glpython hint that it probably should `return` instead. Texts: `runTests`, `testsRunning`, `testsSummary`, `testsNone`, `testsLoadError`, `visualizeExample`, `testsPrintHint`. +**Coverage** (capability `coverage`, off by default): when all examples pass, +the report also lists which lines of the tested file's functions no example +executed, each with its code («Zeile 11: `return "ungenügend"`»; consecutive +lines of one function as «Zeilen 8–12:»). Counted are the statements in +function bodies (also methods and nested functions), as Python's own `trace` +module counts them during the examples - not `def` lines, docstrings, code +outside functions, or other files such as libraries. With the editor, the same +lines are marked in it (line background and a bar in the gutter) until the +file is edited or the program runs again. When every line ran, a neutral note +says so and that this doesn't show whether the examples check the edge cases: +coverage finds untested code, not wrong code. Recorded with `sys.monitoring`, +each line once, so it doesn't slow the examples down. Texts: +`testsUncoveredHeading`, `testsUncoveredLines`, `testsAllCovered`. + Call boxes in the visualizer show a function without its comments and docstring, which don't change what runs; the file itself is shown as written. @@ -295,7 +309,9 @@ https://example.test/embed/?parentOrigin=https%3A%2F%2Fcourse.example Optional query parameters adapt the embed to a host such as the LiaScript macro: `features=run,repl,visualize,…` hides the REPL (`repl`), the visualizer (`visualize`) or its → chain export (`export`) unless listed (capabilities -`repl`, `visualization`, `export`), and `size=large` with +`repl`, `visualization`, `export`); `coverage` in that list switches on the +coverage after Tests (capability `coverage`), which a missing `features` +doesn't; and `size=large` with `visualizerHeight=` shows bigger code and a visualizer at least that tall. The UI language is the optional `lang` query parameter (e.g. the LiaScript @@ -353,14 +369,14 @@ and monospace fonts, no web fonts). A host can override any of them: | Variable | Role | |---|---| | `--glp-bg` | panel background | -| `--glp-surface` | code in the visualizer, output, REPL | +| `--glp-surface` | code in the visualizer, output, REPL; coverage after Tests | | `--glp-border` | decorative borders and separator lines | -| `--glp-border-control` | borders of controls, call boxes in box mode | +| `--glp-border-control` | borders of controls, call boxes in box mode; coverage after Tests (its bar) | | `--glp-text`, `--glp-text-muted` | text and code; line numbers, step counter, secondary info, connector lines | | `--glp-accent`, `--glp-on-accent` | active mode, scrubber, focus, primary button; text on it | | `--glp-current`, `--glp-current-outline` | current subexpression and new value, current line; next step (dashed) | | `--glp-error`, `--glp-error-bg` | errors, failing tests, «Erhalten · Erwartet» | -| `--glp-warning`, `--glp-warning-bg` | hints | +| `--glp-warning`, `--glp-warning-bg` | hints; the coverage marks in the editor | | `--glp-success` | passed tests, runtime ready | | `--glp-syntax-keyword`, `-string`, `-number`, `-comment` | syntax colours (names, operators, brackets use `--glp-text`) | | `--glp-font-code`, `--glp-font-ui`, `--glp-radius` | fonts and corner radius | diff --git a/package.json b/package.json index d22b4d8..ac63184 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:protocol": "npm run build && node scripts/check-embed-protocol.mjs", "test:hints": "node scripts/check-error-hints.mjs", "test:console": "node scripts/check-console-browser.mjs", - "test:runtime": "python3 -B scripts/check-runtime-columns.py && python3 -B scripts/check-runtime-values.py && python3 -B scripts/check-runtime-doctests.py && python3 -B scripts/check-runtime-calls.py && python3 -B scripts/check-runtime-hints.py && python3 -B scripts/check-runtime-fidelity.py && python3 -B scripts/check-runtime-replay.py && python3 -B scripts/check-runtime-imports.py && python3 -B scripts/check-runtime-repl.py", + "test:runtime": "python3 -B scripts/check-runtime-columns.py && python3 -B scripts/check-runtime-values.py && python3 -B scripts/check-runtime-doctests.py && python3 -B scripts/check-runtime-coverage.py && python3 -B scripts/check-runtime-calls.py && python3 -B scripts/check-runtime-hints.py && python3 -B scripts/check-runtime-fidelity.py && python3 -B scripts/check-runtime-replay.py && python3 -B scripts/check-runtime-imports.py && python3 -B scripts/check-runtime-repl.py", "prepare": "npm run build" }, "peerDependencies": { diff --git a/scripts/check-console-browser.mjs b/scripts/check-console-browser.mjs index b743e13..8d49187 100644 --- a/scripts/check-console-browser.mjs +++ b/scripts/check-console-browser.mjs @@ -4,6 +4,9 @@ // the second report used to land out of view below the first one). Tests // starts as Run does, so the console then holds only the new report, and // the REPL has the definitions just tested (not the ones of the last Run). +// With "coverage" in features (P23), a report that all passed lists the +// lines no example executed - which also checks that Pyodide's Python has +// sys.monitoring - and without it, or with a failing example, it doesn't. // // Drives the built embed (dist-embed/, from npm run build:embed) in Chrome // through its protocol, as the LiaScript macro does; Pyodide comes from its @@ -21,7 +24,8 @@ const stopServer = () => server.kill() process.on('exit', stopServer) // The parent page, served on the embed's own origin: it loads the embed and -// passes messages on, version 4. +// passes messages on, version 4. Its own query (e.g. "?features=…") goes on +// to the embed. const harness = ` ` const program = (examples) => `def schnitt(a, b):\n """\n${examples.map(([call, want]) => ` >>> ${call}\n ${want}\n`).join('')} """\n return (a + b) / 2\n` @@ -113,6 +117,35 @@ try { console.error(error.message) } await page.close() + + // Coverage after Tests. + const bewertung = `def bewertung(note):\n """\n >>> bewertung(5)\n 'genügend'\n >>> bewertung(6)\n 'genügend'\n """\n if note >= 4:\n return "genügend"\n else:\n return "ungenügend"\n` + const istGenuegend = `def ist_genuegend(note):\n """\n >>> ist_genuegend(5.5)\n True\n >>> ist_genuegend(3.5)\n False\n """\n return note >= 4\n` + const bewertungFailing = bewertung.replace(">>> bewertung(6)\n 'genügend'", ">>> bewertung(6)\n 'gut'") + for (const [label, query, source, expected] of [ + ['coverage: a line no example ran', '?features=run,repl,visualize,tests,coverage', bewertung, 'Von keinem Beispiel ausgeführt:\nZeile 11: return "ungenügend"'], + ['coverage: every line ran', '?features=run,repl,visualize,tests,coverage', istGenuegend, 'Jede Zeile der Funktionen wurde von mindestens einem Beispiel ausgeführt. Ob die Beispiele auch die Grenzfälle prüfen, zeigt das nicht.'], + ['coverage: not with a failing example', '?features=run,repl,visualize,tests,coverage', bewertungFailing, null], + ['coverage: not without it in features', '?features=run,repl,visualize,tests', bewertung, null], + ['coverage: not without features', '', bewertung, null], + ]) { + const page = await browser.newPage({ viewport: { width: 800, height: 950 } }) + await page.route((url) => url.pathname === '/check-console.html', (route) => route.fulfill({ contentType: 'text/html', body: harness })) + await page.goto(`${origin}/check-console.html${query}`) + await page.waitForFunction(() => window.ready, null, { timeout: 120000 }) + const frame = page.frameLocator('#embed') + await page.evaluate((source) => window.send('test', source), source) + await frame.locator('.test-summary').waitFor({ timeout: 60000 }) + try { + const coverage = (await frame.locator('.test-coverage').allInnerTexts()).map((text) => text.replace(/\n+/g, '\n')) + assert.deepEqual(coverage, expected === null ? [] : [expected], label) + console.log(`${label}: ok`) + } catch (error) { + failed += 1 + console.error(error.message) + } + await page.close() + } } finally { await browser.close() stopServer() diff --git a/scripts/check-runtime-coverage.py b/scripts/check-runtime-coverage.py new file mode 100644 index 0000000..fe325c0 --- /dev/null +++ b/scripts/check-runtime-coverage.py @@ -0,0 +1,281 @@ +"""Regression test: coverage after Tests (src/runtime/doctests.py, the +"coverage" capability) - which statements in the functions of the tested file +no doctest example executed. Each case is also checked against Python's own +trace module: the lines it counts as executed must leave exactly the same +statements uncovered. + +Runs the runtime with CPython (no Pyodide needed): + python3 -B scripts/check-runtime-coverage.py +""" +import contextlib +import doctest +import io +import json +import os +import sys +import tempfile +import trace +import types + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUNTIME = os.path.join(ROOT, "src", "runtime") + +namespace = {"__name__": "glpython_runtime"} +for name in ("hints.py", "doctests.py"): + with open(os.path.join(RUNTIME, name), encoding="utf-8") as handle: + exec(compile(handle.read(), name, "exec"), namespace) +run_doctests = namespace["_glpython_run_doctests"] +coverage_statements = namespace["_glpython_coverage_statements"] +uncovered_from = namespace["_glpython_uncovered"] + +failures = [] +folder = tempfile.mkdtemp(prefix="glpython-coverage-") +sys.path.insert(0, folder) # the worker mounts a project's files next to each other + + +def write(name, source): + path = os.path.join(folder, name) + with open(path, "w", encoding="utf-8") as handle: + handle.write(source) + return path + + +def run(source, name="main.py", coverage=True): + path = write(name, source) + with contextlib.redirect_stdout(io.StringIO()): + return json.loads(run_doctests(path, coverage)) + + +def by_trace(source, name="main.py"): + """The uncovered statements when Python's trace module counts the lines + the examples executed.""" + import ast + path = write(name, source) + module_name = name.removesuffix(".py") + module = types.ModuleType(module_name) + module.__file__ = path + code = compile(source, path, "exec") + with contextlib.redirect_stdout(io.StringIO()): + exec(code, module.__dict__) + tracer = trace.Trace(count=1, trace=0) + runner = doctest.DocTestRunner(verbose=False) + with contextlib.redirect_stdout(io.StringIO()): + for test in doctest.DocTestFinder().find(module, module_name): + tracer.runfunc(runner.run, test, out=lambda text: None, clear_globs=False) + executed = {line for (filename, line) in tracer.results().counts if filename == path} + return uncovered_from(source, coverage_statements(ast.parse(source), code), executed) + + +def expect(label, source, expected, name="main.py"): + """`expected`: the uncovered entries as (start, end, code).""" + report = run(source, name) + got = [(entry["start"], entry["end"], entry["code"]) for entry in report.get("uncovered", [])] + if got != expected: + failures.append(f"{label}: got {got!r}, expected {expected!r}") + reference = [(entry["start"], entry["end"], entry["code"]) for entry in by_trace(source, name)] + if got != reference: + failures.append(f"{label}: got {got!r}, but Python's trace module counts {reference!r}") + return report + + +# The reference cases. Fully covered, though 4 is missing as an example. +report = expect("ist_genuegend", '''def ist_genuegend(note): + """ + >>> ist_genuegend(5.5) + True + >>> ist_genuegend(3.5) + False + """ + return note >= 4 +''', []) +if report.get("failed") != 0 or report.get("attempted") != 2: + failures.append(f"ist_genuegend: report {report!r}") + +expect("bewertung", '''def bewertung(note): + """ + >>> bewertung(5) + 'genügend' + >>> bewertung(6) + 'genügend' + """ + if note >= 4: + return "genügend" + else: + return "ungenügend" +''', [(11, 11, 'return "ungenügend"')]) + +expect("vorzeichen", '''def vorzeichen(x): + """ + >>> vorzeichen(3) + 1 + >>> vorzeichen(-2) + -1 + """ + if x > 0: + return 1 + elif x < 0: + return -1 + return 0 +''', [(12, 12, "return 0")]) + +# A library (another file of the project) is never reported, even where no +# example reaches it. +write("noten.py", '''def runde(note): + if note < 1: + return 1 + return round(note * 2) / 2 +''') +expect("library", '''from noten import runde + +def schnitt(noten): + """ + >>> schnitt([4, 5]) + 4.5 + """ + return runde(sum(noten) / len(noten)) +''', []) + +# Only the examples count: the file's own top-level code calling a function +# doesn't, and neither does the code outside functions. +expect("top-level code", '''def gross(x): + """ + >>> gross(5) + True + """ + if x > 3: + return True + return False + +print(gross(1)) +for i in range(2): + pass +''', [(8, 8, "return False")]) + +# A function no example reaches: its whole body as one entry (def line and +# docstring left out); consecutive uncovered statements are grouped, a +# covered one in between splits them. +expect("whole function", '''def doppelt(x): + """ + >>> doppelt(2) + 4 + """ + return 2 * x + +def hilfe(x): + """Ohne Beispiele.""" + y = x + 1 + if y > 2: + y = y * 2 + return y +''', [(10, 13, "y = x + 1\nif y > 2:\n y = y * 2\nreturn y")]) + +expect("groups", '''def stufe(x): + """ + >>> stufe(1) + 'klein' + """ + if x > 10: + a = "gross" + return a + b = "klein" + if x < 0: + b = "negativ" + return b +''', [(7, 8, 'a = "gross"\nreturn a'), (11, 11, 'b = "negativ"')]) + +# A statement over several lines counts when any of its lines ran; the +# header of a compound statement counts for itself, its body separately. +expect("several lines", '''def summe(werte, + start=0): + """ + >>> summe([1, 2]) + 3 + """ + total = ( + start + ) + for wert in ( + werte + ): + total = total + wert + while (total + < 0): + total = (total + + 1) + return total +''', [(16, 17, "total = (total\n + 1)")]) + +# Statements without bytecode of their own are never reported; neither is +# the docstring, nor a nested function's def line (its body counts). +expect("no bytecode", '''zaehler = 0 + +def zaehle(n): + """ + >>> zaehle(1) + 1 + """ + global zaehler + zaehler = zaehler + n + while True: + break + def innen(): + return 0 + return zaehler +''', [(13, 13, "return 0")]) + +# try/except, methods, decorators and lambdas. +expect("try and methods", '''import functools + +class Konto: + """ + >>> Konto(5).abheben(2) + 3 + """ + def __init__(self, stand): + self.stand = stand + + @functools.cache + def zins(self): + return 0 + + def abheben(self, betrag): + try: + self.stand = self.stand - betrag + except TypeError: + return None + sortiert = sorted([3, 1], key=lambda x: -x) + return self.stand +''', [(13, 13, "return 0"), (19, 19, "return None")]) + +# Failing examples don't stop the recording (the UI shows the coverage only +# when all pass). +report = expect("failing example", '''def f(x): + """ + >>> f(1) + 2 + """ + if x > 0: + return x + return -x +''', [(8, 8, "return -x")]) +if report.get("failed") != 1: + failures.append(f"failing example: report {report!r}") + +# Without the capability: no coverage in the report. +report = run("def f(x):\n '''\n >>> f(1)\n 1\n '''\n return x\n", coverage=False) +if "uncovered" in report: + failures.append(f"without coverage: report {report!r}") + +# The recording is switched off again: a second run counts afresh, and +# sys.monitoring is free. +report = run("def f(x):\n '''\n >>> f(1)\n 1\n '''\n return x\n") +report = run("def f(x):\n '''\n >>> f(1)\n 1\n '''\n return x\n") +if report.get("uncovered") != []: + failures.append(f"second run: report {report!r}") +if sys.monitoring.get_tool(sys.monitoring.COVERAGE_ID) is not None: + failures.append("sys.monitoring's coverage tool is still taken") + +if failures: + print("\n".join(failures)) + sys.exit(f"{len(failures)} coverage check(s) failed.") +print("Runtime coverage checks passed.") diff --git a/src/GlPythonWorkspace.tsx b/src/GlPythonWorkspace.tsx index 2bdfed5..d57378c 100644 --- a/src/GlPythonWorkspace.tsx +++ b/src/GlPythonWorkspace.tsx @@ -1,7 +1,7 @@ import { createPortal } from 'react-dom' import { Fragment, useEffect, useLayoutEffect, useMemo, useRef, useState, type CSSProperties, type KeyboardEvent, type PointerEvent as ReactPointerEvent, type ReactNode } from 'react' import Editor, { useMonaco } from '@monaco-editor/react' -import type { Monaco } from '@monaco-editor/react' +import type { Monaco, OnMount } from '@monaco-editor/react' import { AlertTriangle, BookOpen, Copy, Square, FileCode2, FlaskConical, PanelRightClose, PanelRightOpen, ChevronLeft, ChevronRight, Ellipsis, Play, RotateCcw, SkipBack, SkipForward, Sparkles, SquareTerminal, Terminal, Workflow, X } from 'lucide-react' import { projectImportsGturtle, type Project } from './project' import { GraphicsWindow, type TurtleCommand } from './GraphicsWindow' @@ -96,6 +96,13 @@ type CoreStrings = { testsLoadError?: (message: string) => string visualizeExample?: string testsPrintHint?: (source: string, output: string) => string + // Coverage after Tests (capability `coverage`), when all examples passed: + // the heading above the lines no example executed, each entry's label + // ("Zeile 11:", "Zeilen 8–12:"), and the note when every line ran - which + // must not read as "done": covered code can still be wrong. + testsUncoveredHeading?: string + testsUncoveredLines?: (start: number, end: number) => string + testsAllCovered?: string // Shown under the final value when a failing doctest example is // visualized: the output doctest got and the one the example expected, // e.g. "Got: 4.0 · Expected: 4". @@ -145,6 +152,9 @@ type WorkspaceCapabilities = { tests: boolean // The visualizer's "copy → chain" button and its comments switch. export: boolean + // After Tests, when all examples pass: which lines of the file's functions + // no example executed, in the report and (with the editor) marked in it. + coverage: boolean // The output panel of a visualization whose program prints nothing and // raises no error, showing «(Keine Ausgabe)». Off for visualization-only // embeds, so a short visualization takes less space. @@ -166,6 +176,7 @@ const defaultWorkspaceCapabilities: WorkspaceCapabilities = { repl: true, tests: true, export: true, + coverage: false, emptyOutput: true, closeFileTabs: true, } @@ -175,6 +186,8 @@ type WorkspaceCommand = { type: 'run' | 'visualize' | 'reset' | 'test' | 'stop' } +type CoverageGap = { start: number; end: number; code: string } + // What the worker reports for a Tests run (see doctests.py). type TestReport = { // The file failed while running (its traceback is the program's output), @@ -185,6 +198,9 @@ type TestReport = { // `want`/`got` only for an example whose output didn't match (not one that raised). failures?: Array<{ name: string; line: number; source: string; text: string; want?: string; got?: string; errorHint?: ErrorHint | null }> printHints?: Array<{ source: string; output: string }> + // With the coverage capability: what no example executed, as groups of + // consecutive statements of one function (lines `start`-`end`, their code). + uncovered?: CoverageGap[] // The file didn't load, or doctest couldn't read its examples; with the // hint for that error, if any. error?: string @@ -858,6 +874,7 @@ function TestReportView({ report, t, onVisualizeExample }: { report: TestReport; {attempted === 0 ? (t.testsNone ?? 'No examples found in docstrings.') : (t.testsSummary?.(attempted - failed, attempted) ?? `${attempted - failed} of ${attempted} examples passed`)}

{(report.printHints ?? []).map((hint, index) =>

{t.testsPrintHint?.(hint.source, hint.output) ?? `Note: ${hint.source} printed ${hint.output}, but returned None.`}

)} + {failed === 0 && attempted > 0 && report.uncovered && } {(report.failures ?? []).map((failure, index) => ( 
{failure.text.replace(/^\*+\n/, '').replace(/^\n+|\n+$/g, '')}
@@ -869,6 +886,19 @@ function TestReportView({ report, t, onVisualizeExample }: { report: TestReport; ) } +// Coverage after Tests that all passed: the lines no example executed, each +// with its code, or a neutral note when every line ran. +function TestCoverageView({ gaps, t }: { gaps: CoverageGap[]; t: CoreStrings }) { + if (gaps.length === 0) return

{t.testsAllCovered ?? 'Every line of the functions was run by at least one example. This doesn\'t show whether the examples also check the edge cases.'}

+ const label = (gap: CoverageGap) => t.testsUncoveredLines?.(gap.start, gap.end) ?? (gap.start === gap.end ? `Line ${gap.start}:` : `Lines ${gap.start}\u2013${gap.end}:`) + return ( +
+

{t.testsUncoveredHeading ?? 'Not run by any example:'}

+
    {gaps.map((gap) =>
  • {gap.code.includes('\n') ? <>{label(gap)}
    {gap.code}
    : <>{label(gap)} {gap.code}}
    • )}
+
+ ) +} + // The hint below an error (hints.py), worded by CoreStrings.errorHint, with // the names in it as code. Python's message above it stays as it is. function ErrorHintNote({ hint, t, context = 'other' }: { hint: ErrorHint | null | undefined; t: CoreStrings; context?: ErrorHintContext }) { @@ -1009,6 +1039,20 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { const visualizerCanvasRef = useRef(null) const inputChannelRef = useRef<{ control: Int32Array; payload: Uint8Array } | null>(null) const selectedFile = project.files[activeFile] ?? project.files[project.mainFile] + const selectedPath = project.files[activeFile] ? activeFile : project.mainFile + // The lines the last Tests run left uncovered (see TestCoverageView), + // marked in the editor while that file shows, until it is edited or the + // program runs again. + const [coverageMarks, setCoverageMarks] = useState<{ path: string; gaps: CoverageGap[] } | null>(null) + const testedFileRef = useRef(project.mainFile) + const [normalEditor, setNormalEditor] = useState[0] | null>(null) + useEffect(() => { + // The editor is unmounted (and its model gone) while visualizing. + if (!normalEditor || !monaco || !normalEditor.getModel()) return + const gaps = coverageMarks?.path === selectedPath ? coverageMarks.gaps : [] + const collection = normalEditor.createDecorationsCollection(gaps.map((gap) => ({ range: new monaco.Range(gap.start, 1, gap.end, 1), options: { isWholeLine: true, className: 'coverage-line', linesDecorationsClassName: 'coverage-gutter' } }))) + return () => collection.clear() + }, [normalEditor, monaco, coverageMarks, selectedPath]) const stringsRef = useRef(t) stringsRef.current = t @@ -1041,6 +1085,8 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { const id = ++replIdRef.current revealEntryRef.current = id setConsoleEntries((entries) => [...entries, { kind: 'tests', id, report: event.data as TestReport }]) + const { uncovered, attempted = 0, failed = 0 } = event.data + setCoverageMarks(uncovered && uncovered.length > 0 && failed === 0 && attempted > 0 ? { path: testedFileRef.current, gaps: uncovered } : null) setIsTesting(false) } if (event.data.type === 'repl-result') { @@ -1106,6 +1152,7 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { } const updateCode = (code: string | undefined) => { + if (coverageMarks?.path === activeFile) setCoverageMarks(null) onFileCodeChange(activeFile, code ?? '') } const runCode = () => { @@ -1113,7 +1160,7 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { if (workspaceCapabilities.graphics && projectImportsGturtle(project)) onGraphicsOpenChange(true) runStreamStartedRef.current = false consoleAtBottomRef.current = true - setIsRunning(true); setOutput(t.runningOutput(project.mainFile)); setOutputErrorHint(null); setConsoleEntries([]) + setIsRunning(true); setOutput(t.runningOutput(project.mainFile)); setOutputErrorHint(null); setConsoleEntries([]); setCoverageMarks(null) workerRef.current.postMessage({ type: 'run', project: { @@ -1142,9 +1189,11 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { if (workspaceCapabilities.graphics && projectImportsGturtle(project)) onGraphicsOpenChange(true) runStreamStartedRef.current = false consoleAtBottomRef.current = true - setIsTesting(true); setOutput(t.runningOutput(project.mainFile)); setOutputErrorHint(null); setConsoleEntries([]) + setIsTesting(true); setOutput(t.runningOutput(project.mainFile)); setOutputErrorHint(null); setConsoleEntries([]); setCoverageMarks(null) + testedFileRef.current = project.mainFile workerRef.current.postMessage({ type: 'test', + coverage: workspaceCapabilities.coverage, project: { mainFile: project.mainFile, files: Object.fromEntries(Object.entries(project.files).map(([path, file]) => [path, file.code])), @@ -1189,7 +1238,7 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { restartPython() exitVisualizer() runStreamStartedRef.current = false - setOutput(t.readyOutput); setOutputErrorHint(null); setConsoleEntries([]); setGraphics([]) + setOutput(t.readyOutput); setOutputErrorHint(null); setConsoleEntries([]); setGraphics([]); setCoverageMarks(null) } const exitVisualizer = () => { setExampleExpectation(null) @@ -2032,7 +2081,7 @@ function GlPythonWorkspace(props: GlPythonWorkspaceProps) { const visualizerBarNode =
{visualTraceSteps.length > 0 ? <>
{ setVisualStepIndex(Number(event.target.value)) }} />{errorTraceStep && }
{clampedStepIndex === -1 ? t.visualizerBeforeStart : t.visualizerStepOf(clampedStepIndex + 1, visualTraceSteps.length)}{visualTrace?.truncated && {t.visualizerTruncated}} : {t.visualizerNoSteps}}
{workspaceCapabilities.export && <>}{workspaceCapabilities.exitVisualizer && }
const visualizerWarningNode = <>{isVisualizing && visualTrace?.error && !errorTraceStep &&
{t.visualizerError(visualTrace.error)}
} const editorTabsNode =
{openFiles.map((path) => { const file = project.files[path]; return })}
- const editorWrapNode =
{isVisualizing ?
: null} />{animationsEnabled && closingBoxes.length > 0 &&
{closingBoxes.map((box) =>
)}
}{(connectorPaths.length > 0 || flights.length > 0) && {connectorPaths.map((connector, index) => )}{animationsEnabled && flights.map((flight) => )}}{animationsEnabled && flights.length > 0 &&
{flights.map((flight) => {flight.value})}
}
: { registerGlpythonPython(instance); defineGlpythonTheme(instance) }} value={selectedFile.code} onChange={updateCode} options={{ minimap: { enabled: false }, bracketPairColorization: { enabled: false }, fontSize: 15, lineHeight: 24, padding: { top: 22 }, fontFamily: 'ui-monospace, SFMono-Regular, Menlo, Consolas, monospace', scrollBeyondLastLine: false, smoothScrolling: true, automaticLayout: true, tabSize: 4, insertSpaces: true, detectIndentation: false, placeholder: t.editorPlaceholder ?? 'Type code here' }} />}
+ const editorWrapNode =
{isVisualizing ?
: null} />{animationsEnabled && closingBoxes.length > 0 &&
{closingBoxes.map((box) =>
)}
}{(connectorPaths.length > 0 || flights.length > 0) && {connectorPaths.map((connector, index) => )}{animationsEnabled && flights.map((flight) => )}}{animationsEnabled && flights.length > 0 &&
{flights.map((flight) => {flight.value})}
}
: { registerGlpythonPython(instance); defineGlpythonTheme(instance) }} value={selectedFile.code} onChange={updateCode} onMount={setNormalEditor} options={{ minimap: { enabled: false }, bracketPairColorization: { enabled: false }, fontSize: 15, lineHeight: 24, padding: { top: 22 }, fontFamily: 'ui-monospace, SFMono-Regular, Menlo, Consolas, monospace', scrollBeyondLastLine: false, smoothScrolling: true, automaticLayout: true, tabSize: 4, insertSpaces: true, detectIndentation: false, placeholder: t.editorPlaceholder ?? 'Type code here' }} />}
const consoleNode = <>{!hideOutputPanel &&
{t.output}{runtimeState !== 'ready' && {t.pyodideRuntime}}
{Object.keys(project.files).length > 1 && {t.runsFile(project.mainFile)}}
{ const element = event.currentTarget; consoleAtBottomRef.current = element.scrollHeight - element.scrollTop - element.clientHeight < 24 }}>{isVisualizing ? 
{isAtErrorStep ? <>{visualStdout && !visualStdout.endsWith('\n') ? `${visualStdout}\n` : visualStdout}{visualTrace?.error} : visualStdout || t.noOutput}
: <>
{output}
{consoleEntries.map((entry) => entry.kind === 'tests' ?
: entry.kind === 'note' ?

{entry.text}

: 
 visualizeReplEntry(entry.id) : undefined} busy={isBusy} />
)}}
{workspaceCapabilities.repl &&
{ event.preventDefault(); submitRepl() }}> { setReplInput(event.target.value); setReplHistoryIndex(null) }} onKeyDown={handleReplKeyDown} aria-label={t.replInputLabel ?? 'Python input'} spellCheck={false} autoComplete="off" autoCapitalize="off" />
}
} // In the stacked layout the graphics show only while the program draws - // for a visualization: if it draws at all, so the panel doesn't appear in diff --git a/src/embed/App.tsx b/src/embed/App.tsx index 41b9b3e..821cc68 100644 --- a/src/embed/App.tsx +++ b/src/embed/App.tsx @@ -38,12 +38,14 @@ function createEmbedProject(source: string, files?: GlPythonFile[], mainFile?: s // LiaScript block's macro option: "?visualizer=box". // Optional "?features=run,repl,visualize,…" from the macro's features= // option: which of the embed's own parts to show (here: the REPL, the -// visualizer and the → chain export). Without it, everything shows. -const features = (() => { +// visualizer and the → chain export). Without it, everything shows - except +// "coverage" (after Tests, what no example executed), which a course switches +// on explicitly. +const featureList = (() => { const list = new URLSearchParams(window.location.search).get('features') - const enabled = list === null ? null : new Set(list.split(',').map((feature) => feature.trim())) - return (feature: string) => enabled === null || enabled.has(feature) + return list === null ? null : new Set(list.split(',').map((feature) => feature.trim())) })() +const features = (feature: string) => featureList === null || featureList.has(feature) // "?size=large" (the macro's large option): bigger code, and with // "&visualizerHeight=600" the visualizer at least that many pixels tall (the @@ -201,6 +203,7 @@ export function App() { graphics: mode === 'turtle', repl: mode === 'eval' && features('repl'), export: features('export'), + coverage: featureList?.has('coverage') ?? false, emptyOutput: mode !== 'visual', }} /> diff --git a/src/i18n.ts b/src/i18n.ts index 0593877..b4584e7 100644 --- a/src/i18n.ts +++ b/src/i18n.ts @@ -57,6 +57,9 @@ export const coreStrings: Record = { testsLoadError: (message) => `The file could not be loaded: ${message}`, visualizeExample: 'Visualize example', testsPrintHint: (source, output) => `Note: ${source} printed ${output}, but returned None.`, + testsUncoveredHeading: 'Not run by any example:', + testsUncoveredLines: (start, end) => start === end ? `Line ${start}:` : `Lines ${start}\u2013${end}:`, + testsAllCovered: 'Every line of the functions was run by at least one example. This doesn\'t show whether the examples also check the edge cases.', exampleGotExpected: (got, want) => `Got: ${got} \u00b7 Expected: ${want}`, errorHint: englishErrorHint, hintLabel: 'Hint:', @@ -131,6 +134,9 @@ export const coreStrings: Record = { testsLoadError: (message) => `Die Datei konnte nicht geladen werden: ${message}`, visualizeExample: 'Beispiel visualisieren', testsPrintHint: (source, output) => `Hinweis: ${source} hat ${output} ausgegeben, aber None zurückgegeben.`, + testsUncoveredHeading: 'Von keinem Beispiel ausgeführt:', + testsUncoveredLines: (start, end) => start === end ? `Zeile ${start}:` : `Zeilen ${start}\u2013${end}:`, + testsAllCovered: 'Jede Zeile der Funktionen wurde von mindestens einem Beispiel ausgeführt. Ob die Beispiele auch die Grenzfälle prüfen, zeigt das nicht.', exampleGotExpected: (got, want) => `Erhalten: ${got} \u00b7 Erwartet: ${want}`, errorHint: germanErrorHint, hintLabel: 'Hinweis:', diff --git a/src/pyodide-worker.js b/src/pyodide-worker.js index 702e472..0b553a9 100644 --- a/src/pyodide-worker.js +++ b/src/pyodide-worker.js @@ -247,7 +247,8 @@ self.onmessage = async (event) => { // definitions the REPL then has - and then run the doctests. If the file // itself fails, its traceback (from the Run part) is the answer, and no // report repeats it ("skipped"). What the file printed shows once, from the - // Run part; the report leaves it out. + // Run part; the report leaves it out. With "coverage", the report also has + // what no example executed (see doctests.py). if (event.data.type === 'test') { if (!await runProgram(event.data.project)) { self.postMessage({ type: 'test-result', skipped: true }) @@ -259,7 +260,7 @@ self.onmessage = async (event) => { lastTestMainFile = event.data.project.mainFile pyodide.setStdout({ batched: () => {} }) pyodide.setStderr({ batched: () => {} }) - const result = JSON.parse(String(callPython(pyodide, '_glpython_run_doctests', `/${event.data.project.mainFile}`))) + const result = JSON.parse(String(callPython(pyodide, '_glpython_run_doctests', `/${event.data.project.mainFile}`, event.data.coverage === true))) self.postMessage({ type: 'test-result', ...result }) } catch (error) { self.postMessage({ type: 'test-result', error: String(error) }) diff --git a/src/runtime/doctests.py b/src/runtime/doctests.py index 6a13bd9..3b8dbc4 100644 --- a/src/runtime/doctests.py +++ b/src/runtime/doctests.py @@ -1,7 +1,10 @@ +import ast +import contextlib import doctest import json import re import sys +import textwrap import traceback import types @@ -80,10 +83,127 @@ def _glpython_doctest_blank_hint(error, module, module_name, source): return None return {"kind": "doctest-blank", "line": start + int(match.group(1))} +# Coverage (the "coverage" capability): the statements in the functions of +# the tested file that no example executed, counted by line as Python's own +# trace module counts them. Not counted: def lines, decorators, docstrings, +# class bodies and code outside functions, and statements without bytecode of +# their own (e.g. "global x"). A statement counts as executed when any line of +# it ran; of a compound statement only its header ("if note >= 4:"), since +# its body's statements count for themselves. +def _glpython_statement_header_end(statement): + if isinstance(statement, (ast.If, ast.While)): + return statement.test.end_lineno + if isinstance(statement, (ast.For, ast.AsyncFor)): + return statement.iter.end_lineno + if isinstance(statement, (ast.With, ast.AsyncWith)): + return max(item.context_expr.end_lineno for item in statement.items) + if isinstance(statement, ast.Match): + return statement.subject.end_lineno + if isinstance(statement, (ast.Try, ast.TryStar)): + return statement.lineno + return statement.end_lineno + +def _glpython_code_lines(code): + lines = set() + for constant in code.co_consts: + if isinstance(constant, types.CodeType): + lines.update(line for _, _, line in constant.co_lines() if line is not None) + lines.update(_glpython_code_lines(constant)) + return lines + +# The statements to count, as (function, first line, last line), in the +# file's order; `function` tells the functions apart (nested ones too). +def _glpython_coverage_statements(tree, code): + executable = _glpython_code_lines(code) + statements = [] + def visit(body, function): + for index, statement in enumerate(body): + if index == 0 and isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant) and isinstance(statement.value.value, str): + continue + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + visit(statement.body, statement) + continue + if isinstance(statement, ast.ClassDef): + visit_outside(statement.body) + continue + end = _glpython_statement_header_end(statement) + if any(line in executable for line in range(statement.lineno, end + 1)): + statements.append((function, statement.lineno, end)) + for field in ("body", "orelse", "finalbody"): + inner = getattr(statement, field, None) + if isinstance(inner, list) and inner and isinstance(inner[0], ast.stmt): + visit(inner, function) + for handler in getattr(statement, "handlers", []): + visit(handler.body, function) + for case in getattr(statement, "cases", []): + visit(case.body, function) + # Outside functions only the definitions in it matter. + def visit_outside(body): + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + visit(statement.body, statement) + elif isinstance(statement, ast.ClassDef): + visit_outside(statement.body) + else: + for field in ("body", "orelse", "finalbody"): + inner = getattr(statement, field, None) + if isinstance(inner, list) and inner and isinstance(inner[0], ast.stmt): + visit_outside(inner) + for handler in getattr(statement, "handlers", []): + visit_outside(handler.body) + visit_outside(tree.body) + return statements + +# The statements no example executed, grouped: consecutive ones of the same +# function form one entry, with their lines (dedented) as code. +def _glpython_uncovered(source, statements, executed): + lines = source.splitlines() + groups = [] + previous = None + for function, start, end in statements: + covered = any(line in executed for line in range(start, end + 1)) + if covered: + previous = None + continue + if previous is not None and previous[0] is function: + previous[2] = end + else: + previous = [function, start, end] + groups.append(previous) + return [{"start": start, "end": end, "code": textwrap.dedent("\n".join(lines[start - 1:end])).strip("\n")} for _, start, end in groups] + +# Records the lines of `path` that run while the context is open, with +# sys.monitoring (Python 3.12): each line reports once and is then switched +# off, so a loop costs nothing more. None if sys.monitoring is taken. +@contextlib.contextmanager +def _glpython_record_lines(path): + monitoring = sys.monitoring + tool = monitoring.COVERAGE_ID + try: + monitoring.use_tool_id(tool, "glpython coverage") + except ValueError: + yield None + return + executed = set() + def on_line(code, line): + if code.co_filename == path: + executed.add(line) + return monitoring.DISABLE + try: + monitoring.register_callback(tool, monitoring.events.LINE, on_line) + monitoring.set_events(tool, monitoring.events.LINE) + monitoring.restart_events() + yield executed + finally: + monitoring.set_events(tool, 0) + monitoring.register_callback(tool, monitoring.events.LINE, None) + monitoring.free_tool_id(tool) + # Runs the doctests of one file the way "python -m doctest" does: the file is # loaded as a module named after it (so "if __name__ == '__main__':" blocks -# don't run), then every example in its docstrings runs in order. -def _glpython_run_doctests(main_path): +# don't run), then every example in its docstrings runs in order. With +# `coverage`, the report also lists what no example executed ("uncovered"). +def _glpython_run_doctests(main_path, coverage=False): global _glpython_test_namespace if "/" not in sys.path: sys.path.insert(0, "/") @@ -95,7 +215,8 @@ def _glpython_run_doctests(main_path): try: with open(main_path, "r", encoding="utf-8") as handle: source = handle.read() - exec(compile(source, main_path, "exec"), module.__dict__) + code = compile(source, main_path, "exec") + exec(code, module.__dict__) except BaseException as error: return json.dumps({"error": "".join(traceback.format_exception_only(type(error), error)).strip(), "errorHint": _glpython_error_hint(error)}) @@ -111,15 +232,20 @@ def _glpython_run_doctests(main_path): original_displayhook = sys.__displayhook__ runner._original_display = original_displayhook sys.__displayhook__ = runner.display + executed = None try: - for test in tests: - test.filename = display_name - runner.run(test, out=lambda text: None, clear_globs=False) + with _glpython_record_lines(main_path) if coverage else contextlib.nullcontext() as executed: + for test in tests: + test.filename = display_name + runner.run(test, out=lambda text: None, clear_globs=False) finally: sys.__displayhook__ = original_displayhook - return json.dumps({ + report = { "attempted": runner.tries, "failed": runner.failures, "failures": runner.failed_examples, "printHints": runner.print_hints, - }) + } + if executed is not None: + report["uncovered"] = _glpython_uncovered(source, _glpython_coverage_statements(ast.parse(source), code), executed) + return json.dumps(report) diff --git a/src/style.css b/src/style.css index 9088f3e..a236ddb 100644 --- a/src/style.css +++ b/src/style.css @@ -309,6 +309,17 @@ .error-hint code { font: 600 1em var(--glp-font-code); } .visual-error-hint { padding: 0 8px 4px; } .visual-error-hint .error-hint { margin: 0; } +/* Coverage after Tests: neutral, neither a success nor a warning (covered + code can still be wrong). */ +.test-coverage { margin: 0 0 8px; padding: 6px 10px; color: var(--glp-text); background: var(--glp-surface); border-left: 3px solid var(--glp-border-control); border-radius: var(--glp-radius); font: 13px/1.5 var(--glp-font-ui); } +.test-coverage-heading { margin: 0 0 2px; } +.test-coverage ul { margin: 0; padding: 0; list-style: none; } +.test-coverage code { font: 13px/1.5 var(--glp-font-code); } +.test-report .test-coverage pre { margin: 0 0 4px 1.5em; } +/* The lines in the editor no example executed (the IDE; see coverageMarks), + in the hints' colours: the editor's own background is --glp-surface. */ +.coverage-line { background: var(--glp-warning-bg); } +.coverage-gutter { margin-left: 3px; width: 4px !important; background: var(--glp-warning); } .test-failure { margin: 0 0 10px; padding: 8px 10px; background: var(--glp-error-bg); border-left: 3px solid var(--glp-error); border-radius: var(--glp-radius); } .test-failure .secondary-button { margin-top: 8px; padding: 4px 10px; font-size: 13px; }