From bcd72bd9216897d18f6e23fb09dcb528b4eff088 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Wed, 6 May 2026 06:46:28 -0700 Subject: [PATCH 1/3] chore(A9b): start plan --- .agents/plans/A9b-pm-suite-pattern.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/plans/A9b-pm-suite-pattern.md b/.agents/plans/A9b-pm-suite-pattern.md index fef4e4fb..a1dfe7a5 100644 --- a/.agents/plans/A9b-pm-suite-pattern.md +++ b/.agents/plans/A9b-pm-suite-pattern.md @@ -5,7 +5,7 @@ issue: null pr: null branch: fix/pm-suite-pattern base: main -status: ready +status: in-progress direction: A unlocks: - pm.lua (continuation of A9a) From c366f953b03dd581bf496f5dc643fa68760bcb52 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Wed, 6 May 2026 07:08:33 -0700 Subject: [PATCH 2/3] fix(pattern): thread VM state through gsub callbacks and fix capture order Six related fixes to the Lua pattern engine that together unblock pm.lua through line 166: - gsub callbacks no longer drop their threaded state, so upvalue mutation and table writes inside `function (c) ... end` replacements survive. - Captures are returned in opening order even when nested. - Position captures `()` render as digits when interpolated via `%n`. - `%1` aliases the whole match when the pattern has no captures. - `%z` and `%Z` character classes match the byte 0 and its complement. - `gsub` and `gmatch` honour Lua 5.3.3's empty-match-skip rule. Plan: .agents/plans/A9b-pm-suite-pattern.md --- lib/lua/vm/stdlib/pattern.ex | 241 ++++++++++++++++++++++++++--------- lib/lua/vm/stdlib/string.ex | 35 +++-- test/lua/vm/string_test.exs | 109 ++++++++++++++++ 3 files changed, 308 insertions(+), 77 deletions(-) diff --git a/lib/lua/vm/stdlib/pattern.ex b/lib/lua/vm/stdlib/pattern.ex index ae40138f..c603e288 100644 --- a/lib/lua/vm/stdlib/pattern.ex +++ b/lib/lua/vm/stdlib/pattern.ex @@ -56,15 +56,23 @@ defmodule Lua.VM.Stdlib.Pattern do """ def gmatch(subject, pattern) do {_anchored, pattern_elems} = compile(pattern) - gmatch_from(subject, 0, byte_size(subject), pattern_elems, []) + gmatch_from(subject, 0, byte_size(subject), pattern_elems, [], -1) end - defp gmatch_from(_subject, pos, len, _pattern, acc) when pos > len do + defp gmatch_from(_subject, pos, len, _pattern, acc, _lastmatch) when pos > len do Enum.reverse(acc) end - defp gmatch_from(subject, pos, len, pattern, acc) do + # PUC Lua's `gmatch_aux` skips a match whose end equals the previous match's + # end — this prevents an empty match that fires immediately after a previous + # non-empty match from being returned twice (once at the boundary, once on + # the empty-advance step). + defp gmatch_from(subject, pos, len, pattern, acc, lastmatch) do case match_pattern(subject, pos, pattern, subject) do + {:match, end_pos, _} when end_pos == lastmatch -> + # Same end as previous match — skip and try at next position. + gmatch_from(subject, pos + 1, len, pattern, acc, lastmatch) + {:match, end_pos, captures} -> result = if captures == [] do @@ -75,73 +83,145 @@ defmodule Lua.VM.Stdlib.Pattern do # Advance at least 1 to prevent infinite loop on empty match next_pos = if end_pos == pos, do: pos + 1, else: end_pos - gmatch_from(subject, next_pos, len, pattern, [result | acc]) + gmatch_from(subject, next_pos, len, pattern, [result | acc], end_pos) :nomatch -> - gmatch_from(subject, pos + 1, len, pattern, acc) + gmatch_from(subject, pos + 1, len, pattern, acc, lastmatch) end end @doc """ - Global substitution. Returns {result_string, count}. - `repl` is one of: binary (string), function, or table-like map. + Global substitution. Returns `{result_string, count}`. + + `repl` is one of: binary (string), function (arity 1, `args -> result`), or + any other term (treated as a no-op, returning the whole match). + + This entry point does NOT thread Lua VM state through the function + replacement — any side effects in the callback (upvalue mutation, table + writes) are dropped on the floor. For Lua-level `string.gsub` use + `gsub_stateful/5` instead. """ def gsub(subject, pattern, repl, max_n \\ nil) do {_anchored, pattern_elems} = compile(pattern) - gsub_from(subject, 0, byte_size(subject), pattern_elems, repl, max_n, 0, []) + + stateful_repl = + if is_function(repl, 1) do + fn args, state -> {repl.(args), state} end + else + repl + end + + {result, count, _state} = + gsub_from(subject, 0, byte_size(subject), pattern_elems, stateful_repl, max_n, 0, [], nil, false) + + {result, count} end - defp gsub_from(subject, pos, len, _pattern, _repl, max_n, count, acc) + @doc """ + Stateful global substitution. Returns `{result_string, count, state}`. + + When `repl` is a function it must have arity 2 — `(args, state) -> {result, state}` + — so callbacks that mutate Lua state (upvalues, tables) thread their + changes back out. String and table replacements are state-pass-through. + """ + def gsub_stateful(subject, pattern, repl, state, max_n \\ nil) do + {_anchored, pattern_elems} = compile(pattern) + gsub_from(subject, 0, byte_size(subject), pattern_elems, repl, max_n, 0, [], state, false) + end + + # Lua 5.3.3+ semantics: an empty match that starts where the *previous* + # match ended (with the previous match itself being empty-and-skipped, or + # immediately following any prior match) does not replace — we advance by + # one byte instead. Without this, ` *` against `a b cd` double-fires at + # every space boundary and produces `-a--b--c-d-` instead of `-a-b-c-d-`. + # + # The flag `skip_empty?` is set after every matched-and-applied replacement; + # the next iteration is allowed to fire an empty match only after we've + # advanced past that boundary. + + defp gsub_from(subject, pos, len, _pattern, _repl, max_n, count, acc, state, _skip_empty?) when pos > len or (max_n != nil and count >= max_n) do # Append remaining subject (clamp pos so we never read past end_of_string) remaining = if pos < len, do: binary_part(subject, pos, len - pos), else: "" - {IO.iodata_to_binary([Enum.reverse(acc), remaining]), count} + {IO.iodata_to_binary([Enum.reverse(acc), remaining]), count, state} end - defp gsub_from(subject, pos, len, pattern, repl, max_n, count, acc) do + defp gsub_from(subject, pos, len, pattern, repl, max_n, count, acc, state, skip_empty?) do case match_pattern(subject, pos, pattern, subject) do + {:match, end_pos, _captures} when skip_empty? and end_pos == pos -> + # Empty match immediately after a previous match — skip without + # replacing, advance by one byte (or terminate at end of subject). + if pos < len do + char = <<:binary.at(subject, pos)>> + gsub_from(subject, pos + 1, len, pattern, repl, max_n, count, [char | acc], state, false) + else + {IO.iodata_to_binary(Enum.reverse(acc)), count, state} + end + {:match, end_pos, captures} -> if max_n != nil and count >= max_n do remaining = binary_part(subject, pos, len - pos) - {IO.iodata_to_binary([Enum.reverse(acc), remaining]), count} + {IO.iodata_to_binary([Enum.reverse(acc), remaining]), count, state} else whole_match = binary_part(subject, pos, max(end_pos - pos, 0)) - replacement = apply_replacement(repl, whole_match, captures) - next_pos = if end_pos == pos, do: pos + 1, else: end_pos - prefix = if end_pos == pos and pos < len, do: <<:binary.at(subject, pos)>>, else: "" - gsub_from(subject, next_pos, len, pattern, repl, max_n, count + 1, [prefix, replacement | acc]) + {replacement, state} = apply_replacement(repl, whole_match, captures, state) + empty_match? = end_pos == pos + next_pos = if empty_match?, do: pos + 1, else: end_pos + prefix = if empty_match? and pos < len, do: <<:binary.at(subject, pos)>>, else: "" + + # Only flag skip on the next iteration if THIS match was non-empty + # — the skip-empty rule guards against an empty match that fires + # immediately on the boundary of a prior non-empty match (see + # Lua 5.3.3 changelog). Empty matches that already advanced via + # `prefix` don't trigger the guard. + gsub_from( + subject, + next_pos, + len, + pattern, + repl, + max_n, + count + 1, + [prefix, replacement | acc], + state, + not empty_match? + ) end :nomatch -> if pos < len do char = <<:binary.at(subject, pos)>> - gsub_from(subject, pos + 1, len, pattern, repl, max_n, count, [char | acc]) + gsub_from(subject, pos + 1, len, pattern, repl, max_n, count, [char | acc], state, false) else - {IO.iodata_to_binary(Enum.reverse(acc)), count} + {IO.iodata_to_binary(Enum.reverse(acc)), count, state} end end end - defp apply_replacement(repl, whole_match, captures) when is_binary(repl) do + defp apply_replacement(repl, whole_match, captures, state) when is_binary(repl) do # Replace %0 with whole match, %1-%9 with captures - replace_captures(repl, whole_match, captures) + {replace_captures(repl, whole_match, captures), state} end - defp apply_replacement(repl, whole_match, captures) when is_function(repl) do + defp apply_replacement(repl, whole_match, captures, state) when is_function(repl, 2) do args = if captures == [], do: [whole_match], else: captures + {result, state} = repl.(args, state) + + replacement = + case result do + result when is_binary(result) -> result + result when is_number(result) -> to_string(result) + false -> whole_match + nil -> whole_match + _ -> whole_match + end - case repl.(args) do - result when is_binary(result) -> result - result when is_number(result) -> to_string(result) - false -> whole_match - nil -> whole_match - _ -> whole_match - end + {replacement, state} end - defp apply_replacement(_repl, whole_match, _captures), do: whole_match + defp apply_replacement(_repl, whole_match, _captures, state), do: {whole_match, state} defp replace_captures("", _whole, _captures), do: "" @@ -149,13 +229,21 @@ defmodule Lua.VM.Stdlib.Pattern do idx = c - ?0 value = - if idx == 0 do - whole - else - Enum.at(captures, idx - 1) || "" + cond do + idx == 0 -> + whole + + # Lua quirk: if the pattern has no captures, %1 refers to the + # whole match (PUC-Lua compatibility). Beyond %1 in that case, + # PUC-Lua errors; we fall back to "" to keep the engine total. + captures == [] and idx == 1 -> + whole + + true -> + Enum.at(captures, idx - 1) || "" end - value <> replace_captures(rest, whole, captures) + capture_to_binary(value) <> replace_captures(rest, whole, captures) end defp replace_captures("%" <> <>, whole, captures) do @@ -166,6 +254,13 @@ defmodule Lua.VM.Stdlib.Pattern do <> <> replace_captures(rest, whole, captures) end + # Captures from `()` (position captures) are integers; everything else is + # already a binary. Coerce to a binary so iodata-flattening downstream + # doesn't reinterpret integers as raw bytes. + defp capture_to_binary(value) when is_binary(value), do: value + defp capture_to_binary(value) when is_integer(value), do: Integer.to_string(value) + defp capture_to_binary(value), do: to_string(value) + # --- Pattern Compiler --- @doc """ @@ -267,15 +362,25 @@ defmodule Lua.VM.Stdlib.Pattern do # Match a compiled pattern against subject starting at pos. # Returns {:match, end_pos, captures} or :nomatch. + # + # Capture model: + # `captures` is a list in OPENING order. Each entry is `{:open, start_pos}` + # while a capture is in-flight, or `{:done, value}` once closed. Position + # captures are appended directly as `{:done, pos+1}`. + # + # `cstack` is a stack of indices into `captures`, pointing at the most + # recently-opened-but-not-yet-closed entry. LIFO — `capture_end` always + # closes the innermost open capture. + # + # This preserves the Lua-required ordering (captures numbered by opening + # position) even for nested groups like `(((.).).* (%w*))`. defp match_pattern(subject, pos, pattern, full_subject) do match_elements(subject, pos, pattern, full_subject, [], []) end # All pattern elements consumed — success - defp match_elements(_subject, pos, [], _full, capture_stack, captures) do - # Close any remaining open captures - final_captures = finalize_captures(capture_stack, captures) - {:match, pos, final_captures} + defp match_elements(_subject, pos, [], _full, _cstack, captures) do + {:match, pos, finalize_captures(captures)} end # Anchor end — must be at end of string @@ -287,20 +392,23 @@ defmodule Lua.VM.Stdlib.Pattern do end end - # Capture start + # Capture start — append open marker, remember its index on the stack defp match_elements(subject, pos, [:capture_start | rest], full, cstack, captures) do - match_elements(subject, pos, rest, full, [{pos, nil} | cstack], captures) + index = length(captures) + match_elements(subject, pos, rest, full, [index | cstack], captures ++ [{:open, pos}]) end # Position capture — record the current 1-based byte position as a number defp match_elements(subject, pos, [:position_capture | rest], full, cstack, captures) do - match_elements(subject, pos, rest, full, cstack, captures ++ [pos + 1]) + match_elements(subject, pos, rest, full, cstack, captures ++ [{:done, pos + 1}]) end - # Capture end - defp match_elements(subject, pos, [:capture_end | rest], full, [{start, nil} | cstack], captures) do + # Capture end — replace the topmost open marker with its captured value + defp match_elements(subject, pos, [:capture_end | rest], full, [index | cstack], captures) do + {:open, start} = Enum.at(captures, index) captured = binary_part(subject, start, pos - start) - match_elements(subject, pos, rest, full, cstack, captures ++ [captured]) + captures = List.replace_at(captures, index, {:done, captured}) + match_elements(subject, pos, rest, full, cstack, captures) end # Greedy star (0 or more, greedy) @@ -339,16 +447,22 @@ defmodule Lua.VM.Stdlib.Pattern do end end - # Backref + # Backref — references a previously-closed capture by 1-based index. + # Open captures and out-of-range references fall through to :nomatch. defp match_elements(subject, pos, [{:backref, n} | rest], full, cstack, captures) do - captured = Enum.at(captures, n - 1, "") - cap_len = byte_size(captured) + case Enum.at(captures, n - 1) do + {:done, captured} when is_binary(captured) -> + cap_len = byte_size(captured) - if pos + cap_len <= byte_size(subject) and - binary_part(subject, pos, cap_len) == captured do - match_elements(subject, pos + cap_len, rest, full, cstack, captures) - else - :nomatch + if pos + cap_len <= byte_size(subject) and + binary_part(subject, pos, cap_len) == captured do + match_elements(subject, pos + cap_len, rest, full, cstack, captures) + else + :nomatch + end + + _ -> + :nomatch end end @@ -497,13 +611,26 @@ defmodule Lua.VM.Stdlib.Pattern do defp match_char_class(ch, ?x), do: ch in ?0..?9 or ch in ?a..?f or ch in ?A..?F defp match_char_class(ch, ?X), do: not (ch in ?0..?9 or ch in ?a..?f or ch in ?A..?F) + # %z — the byte with value 0 (deprecated in 5.2+, present in 5.3 reference); + # %Z — its complement. + defp match_char_class(ch, ?z), do: ch == 0 + defp match_char_class(ch, ?Z), do: ch != 0 + # Escaped literal (non-alphanumeric after %) defp match_char_class(ch, literal), do: ch == literal - defp finalize_captures([], captures), do: captures - - defp finalize_captures([{_start, nil} | _rest], _captures) do - # Unclosed capture — shouldn't happen with valid patterns - [] + # Strip the {:done, value} wrappers in opening order. Any remaining + # {:open, _} entries indicate an unclosed capture, which is a pattern + # validity issue — fail closed by returning []. + defp finalize_captures(captures) do + captures + |> Enum.reduce_while([], fn + {:done, value}, acc -> {:cont, [value | acc]} + {:open, _}, _acc -> {:halt, :unclosed} + end) + |> case do + :unclosed -> [] + acc -> Enum.reverse(acc) + end end end diff --git a/lib/lua/vm/stdlib/string.ex b/lib/lua/vm/stdlib/string.ex index feda46fe..7401a6e9 100644 --- a/lib/lua/vm/stdlib/string.ex +++ b/lib/lua/vm/stdlib/string.ex @@ -736,34 +736,29 @@ defmodule Lua.VM.Stdlib.String do max_n = Enum.at(rest, 0) max_n = if is_number(max_n), do: trunc(max_n) - # Determine replacement function + # Determine stateful replacement: either a binary (literal pattern repl) + # or a 2-arity fn `(args, state) -> {result, state}` so callback side + # effects (upvalue mutation, table writes) thread back out of gsub. repl_fn = cond do is_binary(repl) -> repl - is_function(repl, 1) -> - repl - match?({:tref, _}, repl) -> - # Table replacement: look up match in table - fn [match | _] -> - table = State.get_table(state, repl) - Map.get(table.data, match, match) - end - - match?({:lua_closure, _, _}, repl) -> - fn args -> - {results, _state} = Executor.call_function(repl, args, state) - result = List.first(results) - if result == nil or result == false, do: false, else: result + # Table replacement: look up match in table. Reads only — table + # reads don't mutate state, so we pass it through unchanged. + fn [match | _], st -> + table = State.get_table(st, repl) + value = Map.get(table.data, match, match) + {value, st} end - match?({:native_func, _}, repl) -> - fn args -> - {results, _state} = Executor.call_function(repl, args, state) + match?({:lua_closure, _, _}, repl) or match?({:native_func, _}, repl) -> + fn args, st -> + {results, st} = Executor.call_function(repl, args, st) result = List.first(results) - if result == nil or result == false, do: false, else: result + value = if result == nil or result == false, do: false, else: result + {value, st} end true -> @@ -773,7 +768,7 @@ defmodule Lua.VM.Stdlib.String do expected: "string/function/table" end - {result, count} = Pattern.gsub(s, pattern, repl_fn, max_n) + {result, count, state} = Pattern.gsub_stateful(s, pattern, repl_fn, state, max_n) {[result, count], state} end diff --git a/test/lua/vm/string_test.exs b/test/lua/vm/string_test.exs index 43af8669..d3f0f52a 100644 --- a/test/lua/vm/string_test.exs +++ b/test/lua/vm/string_test.exs @@ -718,6 +718,29 @@ defmodule Lua.VM.StringTest do state = Stdlib.install(State.new()) assert {:ok, [0], _state} = VM.execute(proto, state) end + + test "string.gmatch skips empty match adjacent to previous non-empty match" do + # PUC-Lua's `gmatch_aux` tracks `lastmatch` and skips a match whose + # end position equals the previous match's end. Without this the + # `()%s*()` pattern double-fires at every whitespace run and this + # loop produces spurious extra `-`s in the result. Test taken from + # `pm.lua` lines 169-176 in the Lua 5.3 official test suite. + code = ~S""" + local res = "" + local sub = "a \nbc\t\td" + local i = 1 + for p, e in string.gmatch(sub, "()%s*()") do + res = res .. string.sub(sub, i, p - 1) .. "-" + i = e + end + return res + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["-a-b-c-d-"], _state} = VM.execute(proto, state) + end end describe "string.gsub" do @@ -782,6 +805,80 @@ defmodule Lua.VM.StringTest do state = Stdlib.install(State.new()) assert {:ok, ["hello", 0], _state} = VM.execute(proto, state) end + + # Regression tests for A9b — pattern engine and stateful gsub callbacks. + + test "string.gsub with function callback threads upvalue mutation" do + # Before A9b: the gsub-with-callback path discarded the threaded + # state from each Executor.call_function call, so upvalue mutations + # in the callback (like `count = count + 1`) silently no-op'd. + code = """ + local count = 0 + string.gsub("abc", "[a-c]", function() count = count + 1 end) + return count + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, [3], _state} = VM.execute(proto, state) + end + + test "string.gsub with function callback threads table writes" do + code = """ + local res = {s = ''} + string.gsub("abcde", "[a-c]", function(c) res.s = res.s .. c end) + return res.s + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["abc"], _state} = VM.execute(proto, state) + end + + test "string.gsub returns position-capture as digits in replacement" do + # `()` is a position capture; %1 must render as the integer's digits, + # not as a raw byte (which would corrupt the iodata buffer). + code = ~s{return string.gsub("alo alo", "()[al]", "%1")} + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["12o 56o", 4], _state} = VM.execute(proto, state) + end + + test "string.gsub treats %1 as whole match when pattern has no captures" do + # PUC-Lua compatibility quirk: with no captures, %1 == %0. + code = ~s{return string.gsub("abc", "%w", "%1%0")} + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["aabbcc", 3], _state} = VM.execute(proto, state) + end + + test "string.gsub honours Lua 5.3.3 empty-match-skip rule" do + # Without the skip-empty rule we'd produce `-a--b--c-d-` because the + # empty match fires both at the boundary of every non-empty space match + # and on the next char. + code = ~s{return string.gsub("a b cd", " *", "-")} + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["-a-b-c-d-", 5], _state} = VM.execute(proto, state) + end + + test "string.gsub matches %Z (non-zero byte)" do + code = """ + local s = string.char(0) .. "abc" .. string.char(0) + local result = string.gsub(s, "%Z", "x") + return result, #result + """ + + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, [<<0, ?x, ?x, ?x, 0>>, 5], _state} = VM.execute(proto, state) + end end describe "pattern matching features" do @@ -957,6 +1054,18 @@ defmodule Lua.VM.StringTest do state = Stdlib.install(State.new()) assert {:ok, [5, 5], _state} = VM.execute(proto, state) end + + test "nested captures return in opening order" do + # Captures are numbered by `(` position, not by `)` close order. Before + # A9b's refactor `cstack` accumulated closed captures by close time, + # which reversed nested groups: `(((.).).* (%w*))` returned innermost + # first. + code = ~s|return string.match("alo alo", "^(((.).).* (%w*))$")| + assert {:ok, ast} = Parser.parse(code) + assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + assert {:ok, ["alo alo", "al", "a", "alo"], _state} = VM.execute(proto, state) + end end describe "pattern property tests" do From 83a814cb4d580456e01e1d9079f014022b4750ec Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Wed, 6 May 2026 07:09:51 -0700 Subject: [PATCH 3/3] chore(A9b): mark plan as review --- .agents/plans/A9b-pm-suite-pattern.md | 104 +++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/.agents/plans/A9b-pm-suite-pattern.md b/.agents/plans/A9b-pm-suite-pattern.md index a1dfe7a5..311578af 100644 --- a/.agents/plans/A9b-pm-suite-pattern.md +++ b/.agents/plans/A9b-pm-suite-pattern.md @@ -2,10 +2,10 @@ id: A9b title: Fix pattern-engine character classes blocking pm.lua line 122+ issue: null -pr: null +pr: 190 branch: fix/pm-suite-pattern base: main -status: in-progress +status: review direction: A unlocks: - pm.lua (continuation of A9a) @@ -87,4 +87,102 @@ mix test test/lua/vm/string_test.exs ## Discoveries -(populated during implementation) +The plan's "minimal repro" was misleading: `string.gsub("abc", "[a-c]", "X")` +already worked end-to-end. The character-class compiler and matcher were +fine. The actual bug was that the **gsub-with-function-callback path +discarded the threaded VM state**. The lambda built in `string.ex` for +Lua-closure / native-func replacements called `Executor.call_function` +and pattern-matched `{results, _state}`, throwing away the new state. So +upvalue mutations and table writes inside the callback silently no-op'd. +That explains why `string.gsub(s, '[a-c]', function(c) res.s = res.s .. c end)` +returned `""`: the callback ran, but its state changes never reached +the surrounding script. + +Fixes landed in this PR (in roughly the order they were uncovered): + +1. **State threading through `gsub` callbacks.** Added + `Pattern.gsub_stateful/5` that takes a 2-arity `(args, state) -> + {value, state}` replacement and threads `state` through every match. + `string.ex` now uses this for closure/native/table replacements; + string replacements stay state-pass-through. The legacy `Pattern.gsub/4` + API is preserved for any external callers but documented as + non-state-threading. Resolves SC1/SC2/SC3 immediately. + +2. **`%z` and `%Z` character classes.** Lua 5.3's reference manual + carries them over from 5.1; PUC-Lua treats `%z` as the byte 0 and + `%Z` as its complement. Our matcher had no clause and fell through + to the literal-character case, so `%Z` matched literal `Z`. Added + the two clauses to `match_char_class/2`. This unblocked pm.lua + line 131 (`strset('%Z') == strset('[\1-\255]')`). + +3. **Capture ordering for nested groups.** Captures are numbered by + the position of the opening `(`, but the matcher accumulated closed + captures by close time — so `(((.).).* (%w*))` returned innermost + first. Refactored: `captures` is now a list in opening order with + entries tagged `{:open, start_pos}` while in flight and + `{:done, value}` once closed. `cstack` holds indices into that list, + so capture-end can look up and replace by index. Position captures + record `{:done, pos+1}` directly. Updated backref to match the new + wrapper. This fixed pm.lua line 140. + +4. **Position captures in replacement strings.** `replace_captures` + appended capture values via binary `<>`, but position captures are + integers. Erlang iodata flattening would have reinterpreted them + as raw bytes. Added `capture_to_binary/1` that converts integers + to digit strings. Fixed pm.lua line 155. + +5. **`%1` with no captures means whole match.** PUC-Lua quirk: when + the pattern has no captures, `%1` aliases `%0`. We were returning + `""`. Special-cased `idx == 1 and captures == []`. Fixed pm.lua + line 158. + +6. **Lua 5.3.3 empty-match-skip rule for `gsub`.** After a non-empty + match, an empty match starting at the same position must not fire + a replacement — otherwise `string.gsub("a b cd", " *", "-")` + produces `"-a--b--c-d-"` instead of `"-a-b-c-d-"`. Threaded a + `skip_empty?` flag through `gsub_from`. Set the flag only after + a non-empty match (so a leading empty match like `^` still fires + at end-of-string). Fixed pm.lua line 167. + +7. **`gmatch` lastmatch tracking.** PUC's `gmatch_aux` records the end + position of the last returned match and skips a subsequent match + whose end equals that. Without it, `()%s*()` over `"a \nbc\t\td"` + returned 7 results instead of 5 and the test loop produced + `"-a--b-c--d-"`. Added a `lastmatch` parameter to `gmatch_from`. + +After these fixes pm.lua progresses from line 87 (where it failed +before A9b) past line 166. **It then hits a separate VM bug**: a +`do` block containing a generic `for ... in ... do ... end` loop +raises "bad argument in arithmetic expression" before the loop body +runs. The same loop outside a `do` block works fine. This is not a +pattern-engine issue and is out of scope for A9b; it should be split +into A9c. + +Minimal repro of the follow-up bug: + +```lua +do + for k, v in pairs({a=1, b=2}) do + print(k, v) + end +end +``` + +## What changed + +PR: [#190](https://github.com/tv-labs/lua/pull/190) + +Files touched: +- `lib/lua/vm/stdlib/pattern.ex` — six pattern-engine fixes (state- + threading gsub, %z/%Z, capture order, position-capture coercion, + %1-with-no-captures, empty-match-skip rule for both gsub and gmatch). +- `lib/lua/vm/stdlib/string.ex` — switch to `Pattern.gsub_stateful/5` + for closure / native / table replacements. +- `test/lua/vm/string_test.exs` — eight new regression tests covering + each fix. + +Suite delta: pm.lua remains in `@skipped_tests` (4/24 ready, unchanged) +until A9c lands. Unit tests: 1346 → 1354, 0 regressions. + +Follow-up: A9c should investigate the `do`-block + `for-in` VM bug +documented above before pm.lua can move to `@ready_tests`.