From 8bda88bb7b5d9db83413d7455b25d2f4ce6e3a7d Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Tue, 2 Jun 2026 16:07:04 -0700 Subject: [PATCH 1/2] perf(vm): inline per-call depth check on hot call/return paths PR #283 added per-call recursion-depth bookkeeping to every Lua call/return: a `State.check_call_depth!/1` cross-module call plus a `call_depth + 1` / `- 1` counter update. `fibonacci` (~2.7M near-empty calls) pays this with nothing to amortize against, which is the bulk of the depth-attributable slice of the rc.0 -> rc.1 deep-recursion regression. Replace the cross-module guard with a module-local `next_call_depth!/1` marked `@compile {:inline}` in both `Lua.VM.Dispatcher` and `Lua.VM.Executor`, so the common `:infinity` (default) case collapses to a plain integer increment with no call setup. The helper also returns the already-incremented depth, so the call sites stop reading `call_depth` a second time to bump it. `State.next_call_depth!/1` remains the authoritative definition the inlined copies mirror. The depth LIMIT is unchanged: every finite `max_call_depth` still raises a catchable "stack overflow" at exactly the same frame, and the counter unwinds identically on return and after a caught error. Full suite + recursion_depth_test + error_gallery_test pass. Closes #324 --- lib/lua/vm/dispatcher.ex | 38 ++++++++++++++++++++++++++------------ lib/lua/vm/executor.ex | 21 +++++++++++++++++---- lib/lua/vm/state.ex | 29 ++++++++++++++++++----------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/lib/lua/vm/dispatcher.ex b/lib/lua/vm/dispatcher.ex index 2cd936e5..efea5ca6 100644 --- a/lib/lua/vm/dispatcher.ex +++ b/lib/lua/vm/dispatcher.ex @@ -30,6 +30,20 @@ defmodule Lua.VM.Dispatcher do alias Lua.VM.Table alias Lua.VM.Value + # Per-call depth check, inlined to a plain integer comparison so the + # call-dense recursive path does not pay a cross-module call on every + # Lua call/return. Mirrors `State.next_call_depth!/1`: returns the + # bumped depth, or raises a catchable "stack overflow" once the limit + # is hit. The `:infinity` head (the default) skips the comparison. + @compile {:inline, next_call_depth!: 1} + + defp next_call_depth!(%State{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 + defp next_call_depth!(%State{call_depth: depth, max_call_depth: max}) when depth < max, do: depth + 1 + + defp next_call_depth!(%State{call_stack: call_stack}) do + raise RuntimeError, value: "stack overflow", call_stack: call_stack + end + # Mirror of the interpreter's concat ceiling, inlined as a compile-time # constant so the binary-binary fast path stays a single comparison. See # `Lua.VM.Executor.concat_checked/2` for the rationale. @@ -582,12 +596,12 @@ defmodule Lua.VM.Dispatcher do {code, pc + 1, regs, upvalues, proto, cont, :discard, state.open_upvalues} call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) + depth = next_call_depth!(state) state = %{ state | call_stack: [call_info | state.call_stack], - call_depth: state.call_depth + 1, + call_depth: depth, open_upvalues: %{} } @@ -605,8 +619,8 @@ defmodule Lua.VM.Dispatcher do {:lua_closure, _, _} = closure -> args = collect_args(regs, base + 1, arg_count) call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) - state = %{state | call_stack: [call_info | state.call_stack], call_depth: state.call_depth + 1} + depth = next_call_depth!(state) + state = %{state | call_stack: [call_info | state.call_stack], call_depth: depth} {_results, state} = Executor.call_function(closure, args, state) state = %{state | call_stack: tl(state.call_stack), call_depth: state.call_depth - 1} dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames) @@ -635,12 +649,12 @@ defmodule Lua.VM.Dispatcher do {code, pc + 1, regs, upvalues, proto, cont, base, state.open_upvalues} call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) + depth = next_call_depth!(state) state = %{ state | call_stack: [call_info | state.call_stack], - call_depth: state.call_depth + 1, + call_depth: depth, open_upvalues: %{} } @@ -658,8 +672,8 @@ defmodule Lua.VM.Dispatcher do {:lua_closure, _, _} = closure -> args = collect_args(regs, base + 1, arg_count) call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) - state = %{state | call_stack: [call_info | state.call_stack], call_depth: state.call_depth + 1} + depth = next_call_depth!(state) + state = %{state | call_stack: [call_info | state.call_stack], call_depth: depth} {results, state} = Executor.call_function(closure, args, state) state = %{state | call_stack: tl(state.call_stack), call_depth: state.call_depth - 1} @@ -1039,12 +1053,12 @@ defmodule Lua.VM.Dispatcher do frame = {code, pc + 1, regs, upvalues, proto, cont, dest, state.open_upvalues} call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) + depth = next_call_depth!(state) state = %{ state | call_stack: [call_info | state.call_stack], - call_depth: state.call_depth + 1, + call_depth: depth, open_upvalues: %{} } @@ -1062,8 +1076,8 @@ defmodule Lua.VM.Dispatcher do {:lua_closure, _, _} = closure -> args = collect_args(regs, base + 1, total_args) call_info = Executor.dispatcher_call_info(proto, name_hint, 0) - State.check_call_depth!(state) - state = %{state | call_stack: [call_info | state.call_stack], call_depth: state.call_depth + 1} + depth = next_call_depth!(state) + state = %{state | call_stack: [call_info | state.call_stack], call_depth: depth} {results, state} = Executor.call_function(closure, args, state) state = %{state | call_stack: tl(state.call_stack), call_depth: state.call_depth - 1} apply_multi_call_result(result_count, base, results, code, pc + 1, regs, upvalues, proto, state, cont, frames) diff --git a/lib/lua/vm/executor.ex b/lib/lua/vm/executor.ex index 70daa428..e2272357 100644 --- a/lib/lua/vm/executor.ex +++ b/lib/lua/vm/executor.ex @@ -967,8 +967,8 @@ defmodule Lua.VM.Executor do # without going through this branch. args = collect_args(regs, base + 1, total_args) call_info = %{source: proto.source, line: line, name: hint_name(name_hint), namewhat: hint_namewhat(name_hint)} - State.check_call_depth!(state) - state = %{state | call_stack: [call_info | state.call_stack], call_depth: state.call_depth + 1} + depth = next_call_depth!(state) + state = %{state | call_stack: [call_info | state.call_stack], call_depth: depth} {results, state} = Dispatcher.execute(callee_proto, args, callee_upvalues, state) state = %{state | call_stack: tl(state.call_stack), call_depth: state.call_depth - 1} continue_after_call(results, regs, rest, upvalues, proto, state, cont, frames, line, base, result_count) @@ -1005,12 +1005,12 @@ defmodule Lua.VM.Executor do call_info = %{source: proto.source, line: line, name: hint_name(name_hint), namewhat: hint_namewhat(name_hint)} - State.check_call_depth!(state) + depth = next_call_depth!(state) state = %{ state | call_stack: [call_info | state.call_stack], - call_depth: state.call_depth + 1, + call_depth: depth, open_upvalues: %{} } @@ -2175,6 +2175,19 @@ defmodule Lua.VM.Executor do raise RuntimeError, value: "resulting string too large" end + # Per-call depth check, inlined to a plain integer comparison so the + # call-dense recursive path does not pay a cross-module call on every + # Lua call/return. Mirrors `State.next_call_depth!/1`: returns the + # bumped depth, or raises a catchable "stack overflow" once the limit + # is hit. The `:infinity` head (the default) skips the comparison. + @compile {:inline, next_call_depth!: 1} + defp next_call_depth!(%State{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 + defp next_call_depth!(%State{call_depth: depth, max_call_depth: max}) when depth < max, do: depth + 1 + + defp next_call_depth!(%State{call_stack: call_stack}) do + raise RuntimeError, value: "stack overflow", call_stack: call_stack + end + # ── Metamethod support ───────────────────────────────────────────────────── @metamethod_chain_limit 200 diff --git a/lib/lua/vm/state.ex b/lib/lua/vm/state.ex index cbd161f0..9ce04164 100644 --- a/lib/lua/vm/state.ex +++ b/lib/lua/vm/state.ex @@ -9,7 +9,7 @@ defmodule Lua.VM.State do # Call depth tracked as an O(1) counter that moves in lockstep # with `call_stack` — `length(call_stack)` would be O(depth) per # call. `max_call_depth` bounds it; `:infinity` (the default) - # means no limit. See `check_call_depth!/1`. + # means no limit. See `next_call_depth!/1`. call_depth: 0, max_call_depth: :infinity, metatables: %{}, @@ -55,21 +55,28 @@ defmodule Lua.VM.State do end @doc """ - Guards against unbounded recursion. + Returns the next call depth, guarding against unbounded recursion. + + Call it immediately before pushing a frame onto `call_stack`: it both + enforces the limit and computes the incremented `call_depth` in a + single pass, so callers never read `call_depth` a second time to bump + it. Raises a Lua `"stack overflow"` runtime error when the call depth has - reached `max_call_depth`. Call it immediately before pushing a frame - onto `call_stack`. + already reached `max_call_depth`. When `max_call_depth` is `:infinity` + (the default) the limit comparison is skipped — the only work is the + integer increment. - No-op when depth is under the limit or when `max_call_depth` is - `:infinity` (the default). The clauses are ordered so both common cases - resolve in a single function-head match with no struct rebuild. + The hot call/return paths in `Lua.VM.Dispatcher` and `Lua.VM.Executor` + inline this same comparison directly rather than pay a cross-module + call on every Lua call. This function is the authoritative definition + they mirror, and the entry point for any other caller. """ - @spec check_call_depth!(t()) :: :ok - def check_call_depth!(%__MODULE__{max_call_depth: :infinity}), do: :ok - def check_call_depth!(%__MODULE__{call_depth: depth, max_call_depth: max}) when depth < max, do: :ok + @spec next_call_depth!(t()) :: pos_integer() + def next_call_depth!(%__MODULE__{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 + def next_call_depth!(%__MODULE__{call_depth: depth, max_call_depth: max}) when depth < max, do: depth + 1 - def check_call_depth!(%__MODULE__{call_stack: call_stack}) do + def next_call_depth!(%__MODULE__{call_stack: call_stack}) do raise Lua.VM.RuntimeError, value: "stack overflow", call_stack: call_stack end From 429a219ee948f5ce85fe16b9fcb54bf02fe47ff3 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Thu, 4 Jun 2026 04:42:34 -0700 Subject: [PATCH 2/2] test(vm): pin next_call_depth! and cross-reference its inlined copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State.next_call_depth!/1 had no in-repo callers and no direct unit test, and the inlined private copies in Dispatcher and Executor are only validated transitively through end-to-end recursion tests. Add test/lua/vm/state_test.exs covering the bumped value, the under-limit value, the :infinity skip, and the raise-at-limit "stack overflow" — this pins the authoritative copy and gives the three definitions a shared spec. Expand the comment/doc at each of the three definitions so an editor of one copy is pointed at the other two (which are not compiler-checked against it) and at the test that pins the shared semantics. No behavior change; the inlining that is the perf win is preserved. --- lib/lua/vm/dispatcher.ex | 12 +++++++--- lib/lua/vm/executor.ex | 12 +++++++--- lib/lua/vm/state.ex | 5 +++- test/lua/vm/state_test.exs | 49 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 test/lua/vm/state_test.exs diff --git a/lib/lua/vm/dispatcher.ex b/lib/lua/vm/dispatcher.ex index efea5ca6..610388e0 100644 --- a/lib/lua/vm/dispatcher.ex +++ b/lib/lua/vm/dispatcher.ex @@ -32,9 +32,15 @@ defmodule Lua.VM.Dispatcher do # Per-call depth check, inlined to a plain integer comparison so the # call-dense recursive path does not pay a cross-module call on every - # Lua call/return. Mirrors `State.next_call_depth!/1`: returns the - # bumped depth, or raises a catchable "stack overflow" once the limit - # is hit. The `:infinity` head (the default) skips the comparison. + # Lua call/return. Returns the bumped depth, or raises a catchable + # "stack overflow" once the limit is hit. The `:infinity` head (the + # default) skips the comparison. + # + # This is one of three copies of the same logic: the authoritative + # `Lua.VM.State.next_call_depth!/1` and an identical private copy in + # `Lua.VM.Executor`. They are not compiler-checked against each other, + # so any change here must be mirrored in the other two. The shared + # semantics are pinned by `test/lua/vm/state_test.exs`. @compile {:inline, next_call_depth!: 1} defp next_call_depth!(%State{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 diff --git a/lib/lua/vm/executor.ex b/lib/lua/vm/executor.ex index e2272357..6697768a 100644 --- a/lib/lua/vm/executor.ex +++ b/lib/lua/vm/executor.ex @@ -2177,9 +2177,15 @@ defmodule Lua.VM.Executor do # Per-call depth check, inlined to a plain integer comparison so the # call-dense recursive path does not pay a cross-module call on every - # Lua call/return. Mirrors `State.next_call_depth!/1`: returns the - # bumped depth, or raises a catchable "stack overflow" once the limit - # is hit. The `:infinity` head (the default) skips the comparison. + # Lua call/return. Returns the bumped depth, or raises a catchable + # "stack overflow" once the limit is hit. The `:infinity` head (the + # default) skips the comparison. + # + # This is one of three copies of the same logic: the authoritative + # `Lua.VM.State.next_call_depth!/1` and an identical private copy in + # `Lua.VM.Dispatcher`. They are not compiler-checked against each other, + # so any change here must be mirrored in the other two. The shared + # semantics are pinned by `test/lua/vm/state_test.exs`. @compile {:inline, next_call_depth!: 1} defp next_call_depth!(%State{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 defp next_call_depth!(%State{call_depth: depth, max_call_depth: max}) when depth < max, do: depth + 1 diff --git a/lib/lua/vm/state.ex b/lib/lua/vm/state.ex index 9ce04164..f4a5f42d 100644 --- a/lib/lua/vm/state.ex +++ b/lib/lua/vm/state.ex @@ -70,7 +70,10 @@ defmodule Lua.VM.State do The hot call/return paths in `Lua.VM.Dispatcher` and `Lua.VM.Executor` inline this same comparison directly rather than pay a cross-module call on every Lua call. This function is the authoritative definition - they mirror, and the entry point for any other caller. + those two private copies mirror, and the entry point for any other + caller. The copies are not compiler-checked against this one, so any + change to the limit logic here must be applied to all three; the direct + unit test in `test/lua/vm/state_test.exs` pins the shared semantics. """ @spec next_call_depth!(t()) :: pos_integer() def next_call_depth!(%__MODULE__{call_depth: depth, max_call_depth: :infinity}), do: depth + 1 diff --git a/test/lua/vm/state_test.exs b/test/lua/vm/state_test.exs new file mode 100644 index 00000000..72e09645 --- /dev/null +++ b/test/lua/vm/state_test.exs @@ -0,0 +1,49 @@ +defmodule Lua.VM.StateTest do + @moduledoc """ + Pins the authoritative `State.next_call_depth!/1` and, by extension, the + shared semantics the inlined copies in `Lua.VM.Dispatcher` and + `Lua.VM.Executor` must match: the depth bumps by one under the limit, + `:infinity` skips the comparison, and reaching `max_call_depth` raises a + catchable Lua `"stack overflow"`. + """ + use ExUnit.Case, async: true + + alias Lua.VM.RuntimeError + alias Lua.VM.State + + describe "next_call_depth!/1" do + test "returns the incremented depth when under a finite limit" do + state = %State{call_depth: 3, max_call_depth: 10} + + assert State.next_call_depth!(state) == 4 + end + + test "returns the incremented depth when max_call_depth is :infinity" do + state = %State{call_depth: 41, max_call_depth: :infinity} + + assert State.next_call_depth!(state) == 42 + end + + test "skips the limit comparison for :infinity at depth 0 (the default)" do + assert State.next_call_depth!(%State{}) == 1 + end + + test "raises a catchable stack overflow once the depth reaches the limit" do + frames = [%{source: "chunk", line: 1, name: "f", namewhat: ""}] + state = %State{call_depth: 10, max_call_depth: 10, call_stack: frames} + + assert_raise RuntimeError, ~r/stack overflow/, fn -> + State.next_call_depth!(state) + end + end + + test "the raised error carries the stack-overflow value and the call stack" do + frames = [%{source: "chunk", line: 1, name: "f", namewhat: ""}] + state = %State{call_depth: 5, max_call_depth: 5, call_stack: frames} + + error = catch_error(State.next_call_depth!(state)) + + assert %RuntimeError{value: "stack overflow", call_stack: ^frames} = error + end + end +end