From 8ed02f66e5a571a4d9dead9736ed98a929667e23 Mon Sep 17 00:00:00 2001 From: Jon Ruttan Date: Fri, 20 Nov 2026 00:21:19 -0500 Subject: [PATCH 1/2] test: a golden program's tokens and tree tests/golden/shapes.py is a fixed program with one of each layout the tokenizer and the parser decide. D0-golden holds its token stream and the forms the parser emits, one a line, so a change in the engine's reader, a dialect or the parser shows as the lines it moved. A second parse in the same process is held to the same forms. Co-Authored-By: Claude Fable 5.2 --- tests/golden/shapes.py | 24 +++++++ tests/specs/D0-golden.spec.md | 119 ++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 tests/golden/shapes.py create mode 100644 tests/specs/D0-golden.spec.md diff --git a/tests/golden/shapes.py b/tests/golden/shapes.py new file mode 100644 index 0000000..7ca7070 --- /dev/null +++ b/tests/golden/shapes.py @@ -0,0 +1,24 @@ +# shapes.py -- one of each layout the tokenizer and the parser decide +import math + +class Shape: + def __init__(self, name, sides=0): + self.name = name + self.sides = sides + + def describe(self): + return f"{self.name}: {self.sides} sides" + +def area(kind, *dims): + if kind == "circle": + return math.pi * dims[0] ** 2 + elif kind == "rect": + return dims[0] * dims[1] + else: + raise ValueError("unknown " + kind) + +shapes = [Shape(n, s) for n, s in (("triangle", 3), ("square", 4))] +total = 0 +for s in shapes: + total += s.sides # counted +print(total, {s.name: s.sides for s in shapes}, area("rect", 2, 0x10)) diff --git a/tests/specs/D0-golden.spec.md b/tests/specs/D0-golden.spec.md new file mode 100644 index 0000000..f918e16 --- /dev/null +++ b/tests/specs/D0-golden.spec.md @@ -0,0 +1,119 @@ +# golden: one program's tokens and tree + +`tests/golden/shapes.py` is a fixed program with one of each layout the +tokenizer and the parser decide: a comment, an import, a class with methods, +defaults and a star parameter, an if ladder, an f-string, a comprehension over +a tuple target, an augmented assignment, a dict comprehension and a based +number. Its token stream and the forms the parser emits for it are saved +here whole, one a line, so a change to either is seen as the lines it moved. + +A changed line is not a failure of the bundle by itself: it is a change to +what the engine's reader or a dialect hands this parser, or to the parser, +and the diff says which tokens or forms it reached. Re-record a block only +with the change that explains it. + +## the token stream + +### every token of the program, in order + +```python +(%seq + (def %gold-lines + (fn (self l) + (if (null? l) () (%seq (write (first l)) (%seq (newline) (self (rest l))))))) + (%gold-lines (python-tokenize (File read-all "tests/golden/shapes.py")))) +``` +--- +```output +('tok-newline) +('tok-kw "import") +('tok-name "math") +('tok-newline) +('tok-kw "class") +('tok-name "Shape") +('tok-op ":") +('tok-block (('tok-kw "def") ('tok-name "__init__") ('tok-group "(" (('tok-name "self") ('tok-op ",") ('tok-name "name") ('tok-op ",") ('tok-name "sides") ('tok-op "=") ('tok-number "0" 1)) ")") ('tok-op ":") ('tok-block (('tok-name "self") ('tok-op ".") ('tok-name "name") ('tok-op "=") ('tok-name "name") ('tok-newline) ('tok-name "self") ('tok-op ".") ('tok-name "sides") ('tok-op "=") ('tok-name "sides"))) ('tok-kw "def") ('tok-name "describe") ('tok-group "(" (('tok-name "self")) ")") ('tok-op ":") ('tok-block (('tok-kw "return") ('tok-fstring (123 115 101 108 102 46 110 97 109 101 125 58 32 123 115 101 108 102 46 115 105 100 101 115 125 32 115 105 100 101 115)))))) +('tok-kw "def") +('tok-name "area") +('tok-group "(" (('tok-name "kind") ('tok-op ",") ('tok-op "*") ('tok-name "dims")) ")") +('tok-op ":") +('tok-block (('tok-kw "if") ('tok-name "kind") ('tok-op "==") ('tok-string (99 105 114 99 108 101)) ('tok-op ":") ('tok-block (('tok-kw "return") ('tok-name "math") ('tok-op ".") ('tok-name "pi") ('tok-op "*") ('tok-name "dims") ('tok-group "[" (('tok-number "0" 1)) "]") ('tok-op "**") ('tok-number "2" 1))) ('tok-kw "elif") ('tok-name "kind") ('tok-op "==") ('tok-string (114 101 99 116)) ('tok-op ":") ('tok-block (('tok-kw "return") ('tok-name "dims") ('tok-group "[" (('tok-number "0" 1)) "]") ('tok-op "*") ('tok-name "dims") ('tok-group "[" (('tok-number "1" 1)) "]"))) ('tok-kw "else") ('tok-op ":") ('tok-block (('tok-kw "raise") ('tok-name "ValueError") ('tok-group "(" (('tok-string (117 110 107 110 111 119 110 32)) ('tok-op "+") ('tok-name "kind")) ")"))))) +('tok-name "shapes") +('tok-op "=") +('tok-group "[" (('tok-name "Shape") ('tok-group "(" (('tok-name "n") ('tok-op ",") ('tok-name "s")) ")") ('tok-kw "for") ('tok-name "n") ('tok-op ",") ('tok-name "s") ('tok-kw "in") ('tok-group "(" (('tok-group "(" (('tok-string (116 114 105 97 110 103 108 101)) ('tok-op ",") ('tok-number "3" 1)) ")") ('tok-op ",") ('tok-group "(" (('tok-string (115 113 117 97 114 101)) ('tok-op ",") ('tok-number "4" 1)) ")")) ")")) "]") +('tok-newline) +('tok-name "total") +('tok-op "=") +('tok-number "0" 1) +('tok-newline) +('tok-kw "for") +('tok-name "s") +('tok-kw "in") +('tok-name "shapes") +('tok-op ":") +('tok-block (('tok-name "total") ('tok-op "+=") ('tok-name "s") ('tok-op ".") ('tok-name "sides"))) +('tok-name "print") +('tok-group "(" (('tok-name "total") ('tok-op ",") ('tok-group "{" (('tok-name "s") ('tok-op ".") ('tok-name "name") ('tok-op ":") ('tok-name "s") ('tok-op ".") ('tok-name "sides") ('tok-kw "for") ('tok-name "s") ('tok-kw "in") ('tok-name "shapes")) "}") ('tok-op ",") ('tok-name "area") ('tok-group "(" (('tok-string (114 101 99 116)) ('tok-op ",") ('tok-number "2" 1) ('tok-op ",") ('tok-number "0x10" 4)) ")")) ")") +``` + +## the parsed tree + +### every form the parser emits, in order + +```python +(%gold-lines (python-parse (File read-all "tests/golden/shapes.py"))) +``` +--- +```output +('%py-hoist 'py-math '%py-deleted) +('%py-hoist 'py-Shape '%py-deleted) +('%py-hoist 'py-__init__ '%py-deleted) +('%py-hoist 'py-self '%py-deleted) +('%py-hoist 'py-name '%py-deleted) +('%py-hoist 'py-sides '%py-deleted) +('%py-hoist 'py-describe '%py-deleted) +('%py-hoist 'py-area '%py-deleted) +('%py-hoist 'py-pi '%py-deleted) +('%py-hoist 'py-n '%py-deleted) +('%py-hoist 'py-Shape '%py-deleted) +('%py-hoist 'py-shapes '%py-deleted) +('%py-hoist 'py-total '%py-deleted) +('%py-hoist 'py-s '%py-deleted) +('do ('set! 'py-math ('%py-imp '%py-import-call "math" ('lit ()) 0 ('%py-ns-here)))) +('set! 'py-Shape ('%py-class-build '%py-build-class "Shape" ('list) ('fn ('_ '%py-bases) ('let (('%py-crows ('pair () ()))) ('%seq ('do ('%py-crow-put! '%py-crows "__init__" ('let (('%py-d0 0)) ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "__init__" '%py-more ('list "self" "name")) ('%seq ('%py-arity! "__init__" '%py-more 2 1) ('let (('py-self ('List 'ref 0 '%py-more)) ('py-name ('List 'ref 1 '%py-more)) ('py-sides ('python-opt '%py-more 2 '%py-d0))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%seq ('%py-setattr 'py-self "name" 'py-name) ('%py-setattr 'py-self "sides" 'py-sides)) ()))))))) "__init__" ('list "self" "name" "sides") 2 #f () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "__init__" '%py-more ('list "self" "name" "sides")) ('%seq ('%py-arity! "__init__" '%py-more 3 0) ('let (('py-self ('List 'ref 0 '%py-more)) ('py-name ('List 'ref 1 '%py-more)) ('py-sides ('List 'ref 2 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%seq ('%py-setattr 'py-self "name" 'py-name) ('%py-setattr 'py-self "sides" 'py-sides)) ()))))))) "__init__" ("self" "name" "sides") 3 #f () ()))))) ('%py-crow-put! '%py-crows "describe" ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "describe" '%py-more ('list "self")) ('%seq ('%py-arity! "describe" '%py-more 1 0) ('let (('py-self ('List 'ref 0 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%py-return ('python-str-of-x ('%py-fjoin ('list ('%py-fmtfield ('%py-getattr 'py-self "name") "" "") ": " ('%py-fmtfield ('%py-getattr 'py-self "sides") "" "") " sides")))) ()))))))) "describe" ('list "self") 1 #f () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "describe" '%py-more ('list "self")) ('%seq ('%py-arity! "describe" '%py-more 1 0) ('let (('py-self ('List 'ref 0 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%py-return ('python-str-of-x ('%py-fjoin ('list ('%py-fmtfield ('%py-getattr 'py-self "name") "" "") ": " ('%py-fmtfield ('%py-getattr 'py-self "sides") "" "") " sides")))) ()))))))) "describe" ("self") 1 #f () ())))) ()) ('%py-mkclass "Shape" '%py-bases ('python-reverse ('first '%py-crows)))))))) +('set! 'py-area ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "area" '%py-more ('list "kind")) ('let (('py-kind ('List 'ref 0 '%py-more)) ('py-dims ('python-tuple-of-list ('python-drop '%py-more 1)))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 99 105 114 99 108 101)))) ('%py-return ('python-mul ('%py-getattr ('%py-name-live "math" 'py-math) "pi") ('%py-pow ('%py-index 'py-dims 0) 2))) ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 114 101 99 116)))) ('%py-return ('python-mul ('%py-index 'py-dims 0) ('%py-index 'py-dims 1))) ('%py-raise-any ('%py-call 'python-exc-ValueError ('python-add ('python-str-new ('list 117 110 107 110 111 119 110 32)) 'py-kind))))) ())))))) "area" ('list "kind") 1 #t () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "area" '%py-more ('list "kind")) ('let (('py-kind ('List 'ref 0 '%py-more)) ('py-dims ('python-tuple-of-list ('python-drop '%py-more 1)))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 99 105 114 99 108 101)))) ('%py-return ('python-mul ('%py-getattr ('%py-name-live "math" 'py-math) "pi") ('%py-pow ('%py-index 'py-dims 0) 2))) ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 114 101 99 116)))) ('%py-return ('python-mul ('%py-index 'py-dims 0) ('%py-index 'py-dims 1))) ('%py-raise-any ('%py-call 'python-exc-ValueError ('python-add ('python-str-new ('list 117 110 107 110 111 119 110 32)) 'py-kind))))) ())))))) "area" ("kind") 1 #t () ())))) +('set! 'py-shapes ('let (('%py-acc ('pair () ()))) ('%seq ('%py-comp-each ('%py-iter-open ('%py-mktuple ('%py-mktuple ('python-str-new ('list 116 114 105 97 110 103 108 101)) 3) ('%py-mktuple ('python-str-new ('list 115 113 117 97 114 101)) 4))) ('fn ('_ '%py-item) ('let (('%py-unpacked ('%py-unpack '%py-item 2))) ('let (('py-n ('first '%py-unpacked)) ('py-s ('first ('rest '%py-unpacked)))) ('%set-first! '%py-acc ('pair ('%py-call ('%py-name-live "Shape" 'py-Shape) 'py-n 'py-s) ('first '%py-acc))))))) ('python-list-new ('List 'reverse ('first '%py-acc)))))) +('set! 'py-total 0) +(('fn ('self '%py-src) ('let (('%py-item ('%py-iter-pull! '%py-src))) ('if ('same? '%py-item '%py-gen-done) () ('%seq ('set! 'py-s '%py-item) ('%seq ('set! 'py-total ('%py-iadd 'py-total ('%py-getattr 'py-s "sides"))) ('self '%py-src)))))) ('%py-iter-open 'py-shapes)) +('%py-call '%py-print 'py-total ('let (('%py-acc ('%py-mkdict))) ('%seq ('%py-comp-each ('%py-iter-open 'py-shapes) ('fn ('_ 'py-s) ('%py-dset '%py-acc ('%py-getattr 'py-s "name") ('%py-getattr 'py-s "sides")))) '%py-acc)) ('%py-call ('%py-name-live "area" 'py-area) ('python-str-new ('list 114 101 99 116)) 2 16)) +``` + +### a second parse in the same process emits the same forms + +```python +(%gold-lines (python-parse (File read-all "tests/golden/shapes.py"))) +``` +--- +```output +('%py-hoist 'py-math '%py-deleted) +('%py-hoist 'py-Shape '%py-deleted) +('%py-hoist 'py-__init__ '%py-deleted) +('%py-hoist 'py-self '%py-deleted) +('%py-hoist 'py-name '%py-deleted) +('%py-hoist 'py-sides '%py-deleted) +('%py-hoist 'py-describe '%py-deleted) +('%py-hoist 'py-area '%py-deleted) +('%py-hoist 'py-pi '%py-deleted) +('%py-hoist 'py-n '%py-deleted) +('%py-hoist 'py-Shape '%py-deleted) +('%py-hoist 'py-shapes '%py-deleted) +('%py-hoist 'py-total '%py-deleted) +('%py-hoist 'py-s '%py-deleted) +('do ('set! 'py-math ('%py-imp '%py-import-call "math" ('lit ()) 0 ('%py-ns-here)))) +('set! 'py-Shape ('%py-class-build '%py-build-class "Shape" ('list) ('fn ('_ '%py-bases) ('let (('%py-crows ('pair () ()))) ('%seq ('do ('%py-crow-put! '%py-crows "__init__" ('let (('%py-d0 0)) ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "__init__" '%py-more ('list "self" "name")) ('%seq ('%py-arity! "__init__" '%py-more 2 1) ('let (('py-self ('List 'ref 0 '%py-more)) ('py-name ('List 'ref 1 '%py-more)) ('py-sides ('python-opt '%py-more 2 '%py-d0))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%seq ('%py-setattr 'py-self "name" 'py-name) ('%py-setattr 'py-self "sides" 'py-sides)) ()))))))) "__init__" ('list "self" "name" "sides") 2 #f () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "__init__" '%py-more ('list "self" "name" "sides")) ('%seq ('%py-arity! "__init__" '%py-more 3 0) ('let (('py-self ('List 'ref 0 '%py-more)) ('py-name ('List 'ref 1 '%py-more)) ('py-sides ('List 'ref 2 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%seq ('%py-setattr 'py-self "name" 'py-name) ('%py-setattr 'py-self "sides" 'py-sides)) ()))))))) "__init__" ("self" "name" "sides") 3 #f () ()))))) ('%py-crow-put! '%py-crows "describe" ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "describe" '%py-more ('list "self")) ('%seq ('%py-arity! "describe" '%py-more 1 0) ('let (('py-self ('List 'ref 0 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%py-return ('python-str-of-x ('%py-fjoin ('list ('%py-fmtfield ('%py-getattr 'py-self "name") "" "") ": " ('%py-fmtfield ('%py-getattr 'py-self "sides") "" "") " sides")))) ()))))))) "describe" ('list "self") 1 #f () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "describe" '%py-more ('list "self")) ('%seq ('%py-arity! "describe" '%py-more 1 0) ('let (('py-self ('List 'ref 0 '%py-more))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('%py-return ('python-str-of-x ('%py-fjoin ('list ('%py-fmtfield ('%py-getattr 'py-self "name") "" "") ": " ('%py-fmtfield ('%py-getattr 'py-self "sides") "" "") " sides")))) ()))))))) "describe" ("self") 1 #f () ())))) ()) ('%py-mkclass "Shape" '%py-bases ('python-reverse ('first '%py-crows)))))))) +('set! 'py-area ('python-sig! ('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "area" '%py-more ('list "kind")) ('let (('py-kind ('List 'ref 0 '%py-more)) ('py-dims ('python-tuple-of-list ('python-drop '%py-more 1)))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 99 105 114 99 108 101)))) ('%py-return ('python-mul ('%py-getattr ('%py-name-live "math" 'py-math) "pi") ('%py-pow ('%py-index 'py-dims 0) 2))) ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 114 101 99 116)))) ('%py-return ('python-mul ('%py-index 'py-dims 0) ('%py-index 'py-dims 1))) ('%py-raise-any ('%py-call 'python-exc-ValueError ('python-add ('python-str-new ('list 117 110 107 110 111 119 110 32)) 'py-kind))))) ())))))) "area" ('list "kind") 1 #t () ('list) #t ('%py-here) ('lit (('fn ('%py-fn . '%py-more) ('%seq ('%py-need! "area" '%py-more ('list "kind")) ('let (('py-kind ('List 'ref 0 '%py-more)) ('py-dims ('python-tuple-of-list ('python-drop '%py-more 1)))) ('%py-escape ('fn ('_ '%py-return) ('%seq ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 99 105 114 99 108 101)))) ('%py-return ('python-mul ('%py-getattr ('%py-name-live "math" 'py-math) "pi") ('%py-pow ('%py-index 'py-dims 0) 2))) ('if ('%py-truthy ('python-eq 'py-kind ('python-str-new ('list 114 101 99 116)))) ('%py-return ('python-mul ('%py-index 'py-dims 0) ('%py-index 'py-dims 1))) ('%py-raise-any ('%py-call 'python-exc-ValueError ('python-add ('python-str-new ('list 117 110 107 110 111 119 110 32)) 'py-kind))))) ())))))) "area" ("kind") 1 #t () ())))) +('set! 'py-shapes ('let (('%py-acc ('pair () ()))) ('%seq ('%py-comp-each ('%py-iter-open ('%py-mktuple ('%py-mktuple ('python-str-new ('list 116 114 105 97 110 103 108 101)) 3) ('%py-mktuple ('python-str-new ('list 115 113 117 97 114 101)) 4))) ('fn ('_ '%py-item) ('let (('%py-unpacked ('%py-unpack '%py-item 2))) ('let (('py-n ('first '%py-unpacked)) ('py-s ('first ('rest '%py-unpacked)))) ('%set-first! '%py-acc ('pair ('%py-call ('%py-name-live "Shape" 'py-Shape) 'py-n 'py-s) ('first '%py-acc))))))) ('python-list-new ('List 'reverse ('first '%py-acc)))))) +('set! 'py-total 0) +(('fn ('self '%py-src) ('let (('%py-item ('%py-iter-pull! '%py-src))) ('if ('same? '%py-item '%py-gen-done) () ('%seq ('set! 'py-s '%py-item) ('%seq ('set! 'py-total ('%py-iadd 'py-total ('%py-getattr 'py-s "sides"))) ('self '%py-src)))))) ('%py-iter-open 'py-shapes)) +('%py-call '%py-print 'py-total ('let (('%py-acc ('%py-mkdict))) ('%seq ('%py-comp-each ('%py-iter-open 'py-shapes) ('fn ('_ 'py-s) ('%py-dset '%py-acc ('%py-getattr 'py-s "name") ('%py-getattr 'py-s "sides")))) '%py-acc)) ('%py-call ('%py-name-live "area" 'py-area) ('python-str-new ('list 114 101 99 116)) 2 16)) +``` From 37546772c987e2deaadb84f3cfee1ffce28597f9 Mon Sep 17 00:00:00 2001 From: Jon Ruttan Date: Fri, 20 Nov 2026 00:26:34 -0500 Subject: [PATCH 2/2] feat(repl): an error shows the entry it came from, with carets An error at the prompt printed its own line and nothing about where it was. Both loops now print the layout of a CPython traceback's last frame ahead of it: the line the entry began on, its text, and a caret under each character. The entry is what is known -- a token carries no position and an emitted form no line -- so a one-line entry is marked and a block is shown by its header with no carets. A program on standard input is reported by its own line number; an entry typed at the editor is line 1. The two loops share one entry runner. Co-Authored-By: Claude Fable 5.2 --- python/repl.x | 118 ++++++++++++++++++++++++----- tests/gen-harness.sh | 2 + tests/specs/91-repl-editor.spec.md | 76 +++++++++++++++++++ 3 files changed, 176 insertions(+), 20 deletions(-) diff --git a/python/repl.x b/python/repl.x index e770411..765f113 100644 --- a/python/repl.x +++ b/python/repl.x @@ -203,10 +203,104 @@ (if (eq? lines (lit cancel)) lines (Str8 join "\n" lines))) first-line))) +; --- where an entry failed --------------------------------------------------- +; +; An error is reported against the entry that raised it, in the layout of a +; CPython traceback's last frame: the line the entry began on, its text, and +; a caret under each character of it, ahead of the error's own line. The +; entry is the whole of what is known: a token carries no position and an +; emitted form no line. So an entry of one line is marked, and a block is +; shown by its header with no carets, since they would point at a line that +; may not be the one that failed. +; +; Lines are counted where they are read with no editor -- a program on +; standard input -- so the number is the program's own. An entry typed at +; the editor is line 1, as each is at CPython's prompt. +(def %py-lines-read (pair 0 ())) +(def %py-entry-line (pair 1 ())) +(def %py-entry-now (pair "" ())) + +(def %py-read-counted + (fn (_ prompt) + (let ((line (python-read prompt))) + (if (if (symbol? line) #f (not (guard (_ #f) (Line available?)))) + (%set-first! %py-lines-read (+ (first %py-lines-read) 1)) + ()) + line))) + +; An entry begins: its first line, and the line it is on. +(def %py-entry-begin! + (fn (_ line) + (%set-first! %py-entry-now line) + (%set-first! %py-entry-line + (if (= (first %py-lines-read) 0) 1 (first %py-lines-read))))) + +; Past the last byte of S's first N that is not a blank or a carriage return. +(def %py-text-end + (fn (self s n) + (if (> n 0) + (let ((b (python-code-at s (- n 1)))) + (if (match ((= b 32) #t) ((= b 9) #t) (#t (= b 13))) (self s (- n 1)) n)) + 0))) + +; The characters in bytes I to N of S: every byte but a UTF-8 continuation. +(def %py-chars + (fn (self s i n acc) + (if (>= i n) + acc + (let ((b (python-code-at s i))) + (self s (+ i 1) n (if (if (>= b 128) (< b 192) #f) acc (+ acc 1))))))) + +(def %py-carets + (fn (self n acc) (if (> n 0) (self (- n 1) (Str8 append acc "^")) acc))) + +(def %py-entry-report + (fn (_ entry) + (let ((nl (Str8 index-of "\n" entry))) + (let ((text (if (null? nl) entry (Str8 sub 0 nl entry)))) + (let ((n (%py-text-end text (python-byte-len text)))) + (let ((i (%py-past-blanks text 0 n))) + (do + (display " File \"\", line ") + (display (first %py-entry-line)) + (newline) + (display " ") + (display (Str8 sub i (- n i) text)) + (newline) + (if (null? nl) + (do + (display " ") + (display (%py-carets (%py-chars text i n 0) "")) + (newline)) + ())))))))) + +; What both loops print for an error: where, then what. A report that +; cannot be drawn does not take the error's own line with it. +(def %py-entry-error + (fn (_ err) + (do + (guard (_ ()) (%py-entry-report (first %py-entry-now))) + (display "Error: ") + (display err) + (newline)))) + +; One entry, from its first line: the rest of it read, then evaluated, and +; an error from either reported where the entry is (%py-entry-error). +(def %py-entry-run + (fn (_ line more) + (do + (%py-entry-begin! line) + (guard (%py-err (%py-entry-error %py-err)) + (let ((entry (%py-read-entry line more))) + (unless (eq? entry (lit cancel)) + (do + (%set-first! %py-entry-now entry) + (%py-repl-eval entry)))))))) + ; The loop's own way to the next line: the editor when there is a terminal, ; the byte reader otherwise (python/line.x), the prompt drawn either way. (def %py-more-from-loop - (fn (_) (python-read "... "))) + (fn (_) (%py-read-counted "... "))) ; --- the loop ---------------------------------------------------------------- (def %python-repl ()) @@ -225,7 +319,7 @@ (fn (_) ; Through the line editor when there is a terminal (python/line.x), ; which draws the prompt itself; through the byte reader otherwise. - (let ((line (python-read ">>> "))) + (let ((line (%py-read-counted ">>> "))) (match ((eq? line (lit eof)) (%seq (newline) (Sys exit 0))) ; ctrl-c abandons the line being typed. @@ -235,14 +329,7 @@ (Sys exit 0)) (#t (%seq - (guard (%py-err - (%seq - (display "Error: ") - (%seq - (display %py-err) - (newline)))) - (let ((entry (%py-read-entry line %py-more-from-loop))) - (unless (eq? entry (lit cancel)) (%py-repl-eval entry)))) + (%py-entry-run line %py-more-from-loop) (%python-repl-loop))))))) ; --- the editor's way in ----------------------------------------------------- @@ -261,16 +348,7 @@ (match ((= (Str8 length line) 0) ()) ((if (Str8 =? line "quit()") #t (Str8 =? line "exit()")) (Sys exit 0)) - (#t - (let ((entry (%py-read-entry line %py-more-from-editor))) - (unless (eq? entry (lit cancel)) - (guard (%py-err - (%seq - (display "Error: ") - (%seq - (display %py-err) - (newline)))) - (%py-repl-eval entry)))))))) + (#t (%py-entry-run line %py-more-from-editor))))) ; Python's own spelling of the switch: lang("x"). The name arrives as a ; Python str, code points, and Lang wants bytes. diff --git a/tests/gen-harness.sh b/tests/gen-harness.sh index 19be606..1bbe42e 100755 --- a/tests/gen-harness.sh +++ b/tests/gen-harness.sh @@ -74,6 +74,7 @@ cat > "$OUT" < "$OUT" <", line 1 + 1 + + ^^^ Error: syntax: unexpected end of input in expression still here ``` +## where an entry failed + +An error is reported against the entry that raised it: the line the entry +began on, its text, and a caret under each character of it. The entry is the +whole of what is known, so a block is shown by its header and carries no +carets. + +### an error raised while the line runs is marked the same way + +```python +(do + (%py-eval-line "q91 = [1, 2][7]") + (display "still here") (newline)) +``` +--- +```output + File "", line 1 + q91 = [1, 2][7] + ^^^^^^^^^^^^^^^ +Error: index: list index out of range +still here +``` + +### indentation and trailing blanks are not marked, and a character is one caret + +```python +(%py-entry-run "é91 = 1 + " (fn (_) (lit eof))) +``` +--- +```output + File "", line 1 + é91 = 1 + + ^^^^^^^^^ +Error: syntax: unexpected end of input in expression +``` + +### a block is shown by its header, with no carets + +```python +(do + (def %more91 (pair (list " 1 +" "") ())) + (%py-entry-run "for i91 in [1]:" + (fn (_) + (let ((l (first (first %more91)))) + (do (%set-first! %more91 (rest (first %more91))) l))))) +``` +--- +```output + File "", line 1 + for i91 in [1]: +Error: syntax: unexpected end of input in expression +``` + +### a program on standard input is reported by its own line + +The byte loop counts the lines it reads, and an entry begins on the line it +read last. + +```python +(do + (%set-first! %py-lines-read 6) + (%py-entry-run "1 +" (fn (_) (lit eof))) + (%set-first! %py-lines-read 0) + ()) +``` +--- +```output + File "", line 6 + 1 + + ^^^ +Error: syntax: unexpected end of input in expression +``` + ## the registry ### install registers "python" in the platform's registry