diff --git a/.agents/plans/A9a-pm-suite-executor.md b/.agents/plans/A9a-pm-suite-executor.md index ed8c4ce8..3295b053 100644 --- a/.agents/plans/A9a-pm-suite-executor.md +++ b/.agents/plans/A9a-pm-suite-executor.md @@ -2,13 +2,13 @@ id: A9a title: Fix multi-return register expansion exposed by pm.lua issue: null -pr: null +pr: 189 branch: fix/pm-suite-executor base: main -status: ready +status: review direction: A unlocks: - - pm.lua (continuation of A9) + - pm.lua (continuation of A9, blocked behind A9b) --- ## Goal @@ -90,4 +90,50 @@ mix test test/lua/vm/string_test.exs ## Discoveries -(populated during implementation) +The executor fix lands cleanly: `string.char(range(0, 255))` now +returns the expected 256-byte string and pm.lua advances past line 114 +(`assert(string.len(abc) == 256)`). + +The next failure — pm.lua line 122, `strset('[\200-\210]')` — is a +**pattern-engine bug**, explicitly out of scope per this plan's +"Out of scope" section. Minimal repro: `string.gsub("abc", "[a-c]", +"X")` returns `"abc"` instead of `"XXX"`. The gsub character-class +matcher appears to drop matches. + +Split into [`A9b`](.agents/plans/A9b-pm-suite-pattern.md) — `fix/pm-suite-pattern`. +pm.lua remains in `@skipped_tests` until A9b lands. + +**Implementation summary:** + +- Added `ensure_regs_capacity/2` helper in `lib/lua/vm/executor.ex`. + When a multi-return expansion would write past the end of the + caller's register tuple, the helper grows the tuple lazily with a + small headroom (16 extra slots) so back-to-back expansions don't + thrash. The common case (sufficient capacity) remains a single + `tuple_size/1` check. +- Applied at the three call sites: the `-2` and `n > 0` arms of + `do_frame_return/6`, and the matching arms in `continue_after_call/12`. +- Test coverage in `test/lua/vm/call_stack_test.exs` covers four + shapes: 256-value multi-return into a variadic native call, + 100-value into a variadic native call, fixed-count assignment from a + large multi-return (taking only the first N), and table constructor + expansion from a large multi-return. + +## What changed + +PR: #189 + +Files touched: + +- `lib/lua/vm/executor.ex` — `ensure_regs_capacity/2` helper, applied + at the three multi-return write sites (`do_frame_return/6` -2 and + n>0 arms, plus the matching arms in `continue_after_call/12`). +- `test/lua/vm/call_stack_test.exs` — four new tests under the + "multi-return register expansion" describe block. + +Suite delta: pm.lua advances past line 114 but still fails at line 122. +Remains in `@skipped_tests`. Will move to `@ready_tests` when A9b lands. + +Test count: 1342 → 1346 (4 new tests), 0 failures, no regressions. + +Follow-up: [A9b](A9b-pm-suite-pattern.md) — `fix/pm-suite-pattern`. diff --git a/.agents/plans/A9b-pm-suite-pattern.md b/.agents/plans/A9b-pm-suite-pattern.md new file mode 100644 index 00000000..fef4e4fb --- /dev/null +++ b/.agents/plans/A9b-pm-suite-pattern.md @@ -0,0 +1,90 @@ +--- +id: A9b +title: Fix pattern-engine character classes blocking pm.lua line 122+ +issue: null +pr: null +branch: fix/pm-suite-pattern +base: main +status: ready +direction: A +unlocks: + - pm.lua (continuation of A9a) +--- + +## Goal + +Make the second wave of `pm.lua` assertions pass. With A9a's executor +fix landed, `pm.lua` now reaches line 122 where `strset('[\200-\210]')` +fails — the pattern engine appears to drop matches for character-class +patterns over the full byte range (and possibly other character-class +edge cases). Fix the pattern engine bug(s), advance pm.lua, and ideally +move the suite test from `@skipped_tests` to `@ready_tests`. + +## Out of scope + +- Executor / register-tuple work (covered by A9a). +- Lexer escape work (covered by A9). +- Reworking the pattern engine architecturally — make the smallest fix + that gets the failing class of asserts passing. + +## Success criteria + +- [ ] `string.gsub("abcde", "[a-c]", fn)` invokes the callback with + `"a"`, `"b"`, `"c"` — minimal repro of the strset failure. +- [ ] `strset('[a-z]') == "abcdefghijklmnopqrstuvwxyz"` passes. +- [ ] `strset('[\200-\210]')` returns 11 bytes as in PUC-Lua. +- [ ] `pm.lua` passes end-to-end OR the next failure is documented and + split into A9c. +- [ ] If pm.lua passes end-to-end, move it from `@skipped_tests` to + `@ready_tests` in `test/lua53_suite_test.exs`. +- [ ] Unit tests in `test/lua/vm/string_test.exs` (or + `test/lua/vm/stdlib/pattern_test.exs` if it exists) covering the + fixed classes. +- [ ] `mix test` passes (≥ current count, no regressions). + +## Implementation notes + +Minimal repro (confirmed against this branch after A9a): + +```lua +local s = "abcde" +local res = {s=''} +string.gsub(s, '[a-c]', function (c) res.s = res.s .. c end) +-- res.s is "" but should be "abc" +``` + +Even simpler: `string.gsub("abc", "[a-c]", "X")` should return `"XXX"`. + +Investigate `lib/lua/vm/stdlib/pattern.ex` — likely culprits: + +- Character-class set matching (`[...]` ranges) when invoked through + `gsub` rather than `find`/`match`. +- The interaction between the gsub callback path and the captured + range bounds. + +Check: does `string.match("abc", "[a-c]")` already work? If yes, the +bug is gsub-specific. If no, the bug is in core class compilation. + +Cross-reference A9's discoveries (1–5) — the lexer/escape and +position-capture fixes there are now baseline. + +## Verification + +```bash +mix format +mix compile --warnings-as-errors +mix test +mix test --only lua53 +mix test test/lua/vm/string_test.exs +``` + +## Risks + +- Pattern engine has many corner cases; fixing class-set matching may + regress other patterns. Run the existing string tests as guardrails. +- Lua patterns are NOT regexes. Range semantics use byte values, not + Unicode codepoints. + +## Discoveries + +(populated during implementation) diff --git a/lib/lua/vm/executor.ex b/lib/lua/vm/executor.ex index 7eae09f7..7ff3a03c 100644 --- a/lib/lua/vm/executor.ex +++ b/lib/lua/vm/executor.ex @@ -1136,6 +1136,7 @@ defmodule Lua.VM.Executor do -2 -> # Multi-return expansion: place all results into caller regs from base results_list = List.wrap(results) + caller_regs = ensure_regs_capacity(caller_regs, base + length(results_list)) caller_regs = results_list @@ -1152,6 +1153,7 @@ defmodule Lua.VM.Executor do n when n > 0 -> # Fixed count: place first n results into caller regs from base results_list = List.wrap(results) + caller_regs = ensure_regs_capacity(caller_regs, base + n) caller_regs = Enum.reduce(0..(n - 1), caller_regs, fn i, r -> @@ -1175,6 +1177,7 @@ defmodule Lua.VM.Executor do -2 -> results_list = List.wrap(results) + regs = ensure_regs_capacity(regs, base + length(results_list)) regs = results_list @@ -1189,6 +1192,7 @@ defmodule Lua.VM.Executor do n when n > 0 -> results_list = List.wrap(results) + regs = ensure_regs_capacity(regs, base + n) regs = Enum.reduce(0..(n - 1), regs, fn i, r -> @@ -1199,6 +1203,31 @@ defmodule Lua.VM.Executor do end end + # ── ensure_regs_capacity — grow a register tuple when multi-return overflows ─ + # + # The compiler sizes register tuples for the syntactic call site, but a + # multi-return call can produce more values than the caller statically + # reserved (e.g. `string.char(range(0, 255))` returns 256 values from a + # recursive helper). Rather than pre-allocate for the pathological case, + # grow the tuple lazily here and keep the common case to a single + # `put_elem`. `needed_size` is the total number of slots we must be able + # to write to (i.e. base + count), so capacity must be at least that. + defp ensure_regs_capacity(regs, needed_size) when is_tuple(regs) do + current_size = tuple_size(regs) + + if needed_size > current_size do + # Append nil slots with a small headroom so back-to-back expansions + # don't repeatedly reallocate. + extra = needed_size - current_size + 16 + grow_tuple(regs, extra) + else + regs + end + end + + defp grow_tuple(tuple, 0), do: tuple + defp grow_tuple(tuple, n), do: grow_tuple(Tuple.insert_at(tuple, tuple_size(tuple), nil), n - 1) + # ── find_loop_exit — scan cont for the nearest {:loop_exit, _} marker ────── defp find_loop_exit([{:loop_exit, exit_is} | rest_cont]), do: {exit_is, rest_cont} diff --git a/test/lua/vm/call_stack_test.exs b/test/lua/vm/call_stack_test.exs index a78b8905..29aa490f 100644 --- a/test/lua/vm/call_stack_test.exs +++ b/test/lua/vm/call_stack_test.exs @@ -5,6 +5,7 @@ defmodule Lua.VM.CallStackTest do alias Lua.Parser alias Lua.VM alias Lua.VM.State + alias Lua.VM.Stdlib describe "call stack tracking" do test "tracks current line during execution" do @@ -123,4 +124,90 @@ defmodule Lua.VM.CallStackTest do assert final_state.call_stack == [] end end + + describe "multi-return register expansion" do + # Regression test for the executor crash exposed by pm.lua line 112: + # `string.char(range(0, 255))` where `range` is a recursive helper that + # tail-position multi-returns 256 values. The compiler sizes the caller's + # register tuple for the syntactic call site, but multi-return expansion + # can produce many more values than were statically reserved. The + # executor must grow the register tuple before writing the expanded + # results. + + test "recursive multi-return with 256 values feeds string.char" do + code = """ + local function range(i, j) + if i <= j then return i, range(i+1, j) end + end + return string.char(range(0, 255)) + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + + state = Stdlib.install(State.new()) + assert {:ok, [result], _final_state} = VM.execute(proto, state) + + assert is_binary(result) + assert byte_size(result) == 256 + assert :binary.first(result) == 0 + assert :binary.last(result) == 255 + end + + test "recursive multi-return with 100 values feeds variadic native call" do + # Smaller variant, also exercises the grow-tuple path because the + # default register tuple won't be sized for 100 expanded slots. + code = """ + local function range(i, j) + if i <= j then return i, range(i+1, j) end + end + return string.char(range(65, 164)) + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + + state = Stdlib.install(State.new()) + assert {:ok, [result], _final_state} = VM.execute(proto, state) + + assert is_binary(result) + assert byte_size(result) == 100 + end + + test "fixed-count assignment from large multi-return takes only first N" do + code = """ + local function range(i, j) + if i <= j then return i, range(i+1, j) end + end + local a, b, c = range(0, 200) + return a, b, c + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + + state = State.new() + assert {:ok, results, _final_state} = VM.execute(proto, state) + + assert results == [0, 1, 2] + end + + test "table constructor expands large multi-return into all slots" do + code = """ + local function range(i, j) + if i <= j then return i, range(i+1, j) end + end + local t = {range(1, 150)} + return #t, t[1], t[150] + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + + state = State.new() + assert {:ok, results, _final_state} = VM.execute(proto, state) + + assert results == [150, 1, 150] + end + end end