diff --git a/benchmarks/array_vs_map_probe.exs b/benchmarks/array_vs_map_probe.exs new file mode 100644 index 00000000..c2c92370 --- /dev/null +++ b/benchmarks/array_vs_map_probe.exs @@ -0,0 +1,78 @@ +# Probe: Erlang :array vs Elixir map for the access patterns the table +# workloads exercise. This isolates the data-structure lever from the rest +# of the VM so we can decide whether split storage can plausibly close the +# build/sort gap vs luerl before paying the full refactor cost. + +Code.require_file("helpers.exs", __DIR__) + +ns = [100, 1000] + +build_map = fn n -> + Enum.reduce(1..n, %{}, fn i, acc -> Map.put(acc, i, i * i) end) +end + +build_arr = fn n -> + Enum.reduce(1..n, :array.new([{:size, n}, {:fixed, false}, {:default, nil}]), fn i, acc -> + :array.set(i - 1, i * i, acc) + end) +end + +# Pre-build for read/length/sortwrite probes +maps = Map.new(ns, fn n -> {n, build_map.(n)} end) +arrs = Map.new(ns, fn n -> {n, build_arr.(n)} end) + +sum_map = fn m, n -> Enum.reduce(1..n, 0, fn i, s -> s + Map.get(m, i) end) end +sum_arr = fn a, n -> Enum.reduce(1..n, 0, fn i, s -> s + :array.get(i - 1, a) end) end + +# sort write-back: take sorted values list, write each back under key idx +sortwrite_map = fn n -> + vals = Enum.to_list(n..1//-1) + Enum.reduce(Enum.with_index(vals, 1), %{}, fn {v, idx}, acc -> Map.put(acc, idx, v) end) +end + +sortwrite_arr = fn n -> + vals = Enum.to_list(n..1//-1) + base = :array.new([{:size, n}, {:fixed, false}, {:default, nil}]) + Enum.reduce(Enum.with_index(vals, 1), base, fn {v, idx}, acc -> :array.set(idx - 1, v, acc) end) +end + +# sequence_length: map probes 1..n+1; array is just :array.size +seqlen_map = fn m -> + Stream.iterate(1, &(&1 + 1)) |> Enum.find(fn i -> not Map.has_key?(m, i) end) |> Kernel.-(1) +end + +for n <- ns do + Bench.banner("DS probe n=#{n}") + + Benchee.run( + %{ + "build map" => fn -> build_map.(n) end, + "build array" => fn -> build_arr.(n) end + }, + Bench.opts() + ) + + Benchee.run( + %{ + "sum map (read)" => fn -> sum_map.(maps[n], n) end, + "sum array (read)" => fn -> sum_arr.(arrs[n], n) end + }, + Bench.opts() + ) + + Benchee.run( + %{ + "sortwrite map" => fn -> sortwrite_map.(n) end, + "sortwrite array" => fn -> sortwrite_arr.(n) end + }, + Bench.opts() + ) + + Benchee.run( + %{ + "seqlen map (probe)" => fn -> seqlen_map.(maps[n]) end, + "seqlen array (size)" => fn -> :array.size(arrs[n]) end + }, + Bench.opts() + ) +end diff --git a/guides/sandboxing.md b/guides/sandboxing.md index fb96e6e6..e01cf022 100644 --- a/guides/sandboxing.md +++ b/guides/sandboxing.md @@ -94,11 +94,11 @@ These guards are always on and need no configuration. They cover: | Operation | Guard | Error (catchable with `pcall`) | | :-------- | :---- | :----------------------------- | -| `string.rep`, the `..` operator | result larger than ~256 MiB | `resulting string too large` | +| `string.rep`, the `..` operator | result larger than the string ceiling (default ~256 MiB) | `resulting string too large` | | `string.format` width/precision | field wider than 99 | `invalid conversion` | | `table.unpack` | more than 10M results | `too many results to unpack` | | `table.concat`, `table.move` | range wider than 10M | `range too large` | -| `load` (when enabled) | reader returns > ~256 MiB total | `resulting string too large` | +| `load` (when enabled) | reader returns more than the string ceiling in total | `resulting string too large` | ```elixir {[false, message], _} = @@ -111,6 +111,22 @@ message =~ "resulting string too large" Because these surface as ordinary Lua errors, a script can even recover from them with `pcall`. +The string ceiling is configurable via `:max_string_bytes`: + +```elixir +lua = Lua.new(max_string_bytes: 16 * 1024 * 1024) +``` + +> #### Pair the ceiling with your heap cap {: .warning} +> If you run the VM inside a process capped with `:max_heap_size` (see +> below), set `:max_string_bytes` comfortably *below* that cap. The +> default ceiling permits strings large enough that a smaller heap cap +> would have to catch them instead — and `max_heap_size` only kills when +> a garbage collection happens to observe the oversized live set, which +> makes the outcome timing-dependent. With the ceiling under the cap, +> string bombs are always refused deterministically and the heap cap +> stays what it should be: a backstop for what the VM cannot pre-compute. + ## Call depth By default the VM places no limit on call depth, so deeply recursive or diff --git a/lib/lua.ex b/lib/lua.ex index 84d53ae0..47bb84b7 100644 --- a/lib/lua.ex +++ b/lib/lua.ex @@ -100,15 +100,43 @@ defmodule Lua do iex> {[false, message], _lua} = Lua.eval!(lua, "local function f() return f() end return pcall(f)") iex> message =~ "stack overflow" true + + * `:max_string_bytes` - (default 256 MiB) ceiling for any single string the VM will build, + whether via `..`, `string.rep`, or a `load` reader. An oversized result raises a catchable + `"resulting string too large"` error — the size is computed *before* allocating, so the + bomb is refused rather than detected after the fact. Accepts a positive integer. + + When running the VM inside a process capped with `:max_heap_size`, set this comfortably + below the heap cap: the default ceiling permits strings large enough to trip a smaller + heap cap mid-allocation, where the kill depends on garbage-collection timing rather than + a deterministic refusal. + + iex> lua = Lua.new(max_string_bytes: 1024) + iex> {[false, message], _lua} = Lua.eval!(lua, "return pcall(string.rep, \\"x\\", 2048)") + iex> message =~ "resulting string too large" + true """ @spec new(keyword()) :: t() def new(opts \\ []) do - opts = Keyword.validate!(opts, sandboxed: @default_sandbox, exclude: [], debug: false, max_call_depth: :infinity) + opts = + Keyword.validate!(opts, + sandboxed: @default_sandbox, + exclude: [], + debug: false, + max_call_depth: :infinity, + max_string_bytes: Lua.VM.Limits.max_string_bytes() + ) + exclude = Keyword.fetch!(opts, :exclude) debug = Keyword.fetch!(opts, :debug) max_call_depth = validate_max_call_depth!(Keyword.fetch!(opts, :max_call_depth)) + max_string_bytes = validate_max_string_bytes!(Keyword.fetch!(opts, :max_string_bytes)) - state = %{Lua.VM.Stdlib.install(State.new()) | max_call_depth: max_call_depth} + state = %{ + Lua.VM.Stdlib.install(State.new()) + | max_call_depth: max_call_depth, + max_string_bytes: max_string_bytes + } opts |> Keyword.fetch!(:sandboxed) @@ -124,6 +152,13 @@ defmodule Lua do ":max_call_depth must be a positive integer or :infinity, got: #{inspect(other)}" end + defp validate_max_string_bytes!(bytes) when is_integer(bytes) and bytes > 0, do: bytes + + defp validate_max_string_bytes!(other) do + raise ArgumentError, + ":max_string_bytes must be a positive integer, got: #{inspect(other)}" + end + @doc """ Write Lua code that is parsed at compile-time. diff --git a/lib/lua/vm/dispatcher.ex b/lib/lua/vm/dispatcher.ex index 2cd936e5..de8d7e81 100644 --- a/lib/lua/vm/dispatcher.ex +++ b/lib/lua/vm/dispatcher.ex @@ -23,17 +23,14 @@ defmodule Lua.VM.Dispatcher do alias Lua.Compiler.Prototype alias Lua.VM.Executor alias Lua.VM.InternalError - alias Lua.VM.Limits alias Lua.VM.Numeric alias Lua.VM.RuntimeError alias Lua.VM.State alias Lua.VM.Table - alias Lua.VM.Value # Mirror of the interpreter's concat ceiling, inlined as a compile-time # constant so the binary-binary fast path stays a single comparison. See # `Lua.VM.Executor.concat_checked/2` for the rationale. - @max_string_bytes Limits.max_string_bytes() # Opcode tags. These must stay in lockstep with `Lua.Compiler.Bytecode`. # The module-attribute form lets each case branch match a constant @@ -720,6 +717,29 @@ defmodule Lua.VM.Dispatcher do key = :erlang.element(key_reg + 1, regs) case table_val do + {:tref, id} when is_integer(key) and key >= 1 -> + table = :erlang.map_get(id, state.tables) + + case Table.get(table, key) do + nil -> + case :erlang.map_get(:metatable, table) do + nil -> + regs = :erlang.setelement(dest + 1, regs, nil) + dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames) + + _ -> + {value, state} = + Executor.dispatcher_get_table(table_val, key, state, proto, name_hint) + + regs = :erlang.setelement(dest + 1, regs, value) + dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames) + end + + value -> + regs = :erlang.setelement(dest + 1, regs, value) + dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames) + end + {:tref, id} when is_integer(key) or is_binary(key) -> table = :erlang.map_get(id, state.tables) data = :erlang.map_get(:data, table) @@ -789,7 +809,7 @@ defmodule Lua.VM.Dispatcher do nil -> # No __len possible — Lua 5.3 §3.4.7: # on a table # without __len is the border length of the data map. - len = Value.sequence_length(:erlang.map_get(:data, table)) + len = Table.length(table) regs = :erlang.setelement(dest + 1, regs, len) dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames) @@ -1103,7 +1123,7 @@ defmodule Lua.VM.Dispatcher do right = :erlang.element(b + 1, regs) if is_binary(left) and is_binary(right) do - if byte_size(left) + byte_size(right) > @max_string_bytes do + if byte_size(left) + byte_size(right) > state.max_string_bytes do raise RuntimeError, value: "resulting string too large" end diff --git a/lib/lua/vm/display.ex b/lib/lua/vm/display.ex index f93c7335..69595e79 100644 --- a/lib/lua/vm/display.ex +++ b/lib/lua/vm/display.ex @@ -141,7 +141,7 @@ defmodule Lua.VM.Display do defp peek_table(state, id, decode?) do case Map.fetch(state.tables, id) do {:ok, table} -> - data = table.data + data = Lua.VM.Table.to_map(table) if sequence_like?(data) do Enum.map(1..map_size(data), &wrap_value(Map.fetch!(data, &1), state, decode?)) diff --git a/lib/lua/vm/executor.ex b/lib/lua/vm/executor.ex index 70daa428..e20e6179 100644 --- a/lib/lua/vm/executor.ex +++ b/lib/lua/vm/executor.ex @@ -14,7 +14,6 @@ defmodule Lua.VM.Executor do alias Lua.VM.Dispatcher alias Lua.VM.InternalError - alias Lua.VM.Limits alias Lua.VM.Numeric alias Lua.VM.RuntimeError alias Lua.VM.State @@ -356,7 +355,7 @@ defmodule Lua.VM.Executor do case value do {:tref, id} -> table = Map.fetch!(state.tables, id) - Value.sequence_length(table.data) + Table.length(table) v when is_binary(v) -> byte_size(v) @@ -420,7 +419,7 @@ defmodule Lua.VM.Executor do src = proto.source try_binary_metamethod("__concat", left, right, state, fn -> - concat_checked(concat_coerce(left, 0, src), concat_coerce(right, 0, src)) + concat_checked(concat_coerce(left, 0, src), concat_coerce(right, 0, src), state.max_string_bytes) end) end @@ -1351,12 +1350,12 @@ defmodule Lua.VM.Executor do # also concatenate without metamethods. cond do is_binary(left) and is_binary(right) -> - regs = put_elem(regs, dest, concat_checked(left, right)) + regs = put_elem(regs, dest, concat_checked(left, right, state.max_string_bytes)) do_execute(rest, regs, upvalues, proto, state, cont, frames, line) (is_binary(left) or is_number(left)) and (is_binary(right) or is_number(right)) -> src = proto.source - result = concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src)) + result = concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src), state.max_string_bytes) regs = put_elem(regs, dest, result) do_execute(rest, regs, upvalues, proto, state, cont, frames, line) @@ -1365,7 +1364,7 @@ defmodule Lua.VM.Executor do {result, new_state} = try_binary_metamethod("__concat", left, right, state, fn -> - concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src)) + concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src), state.max_string_bytes) end) regs = put_elem(regs, dest, result) @@ -1627,7 +1626,7 @@ defmodule Lua.VM.Executor do case value do {:tref, id} -> table = Map.fetch!(state.tables, id) - Value.sequence_length(table.data) + Table.length(table) v when is_binary(v) -> byte_size(v) @@ -1668,10 +1667,30 @@ defmodule Lua.VM.Executor do key = elem(regs, key_reg) case table_val do + {:tref, id} when is_integer(key) and key >= 1 -> + # Split-storage fast path: dense positive integers live in the array. + table = :erlang.map_get(id, state.tables) + + case Table.get(table, key) do + nil -> + case :erlang.map_get(:metatable, table) do + nil -> + regs = put_elem(regs, dest, nil) + do_execute(rest, regs, upvalues, proto, state, cont, frames, line) + + _ -> + {value, state} = index_value(table_val, key, state, line, proto.source, name_hint) + regs = put_elem(regs, dest, value) + do_execute(rest, regs, upvalues, proto, state, cont, frames, line) + end + + value -> + regs = put_elem(regs, dest, value) + do_execute(rest, regs, upvalues, proto, state, cont, frames, line) + end + {:tref, id} when is_integer(key) or is_binary(key) -> - # Fast path mirroring get_field: integer/string key on a tref. Skip - # normalize_key (no-op for these) and the full index_value pipeline - # when the entry is present or no metatable is set. + # Non-positive integer / string key: read straight from the hash map. table = :erlang.map_get(id, state.tables) case :erlang.map_get(:data, table) do @@ -2162,16 +2181,15 @@ defmodule Lua.VM.Executor do # in one BIF call — faster than the GC-time `max_heap_size` check can # react. Guard the size deterministically here, matching the Layer A # stdlib checks, so the loop fails with a catchable error long before it - # threatens the host. `byte_size/1` is O(1), so this is cheap on the hot - # path. Compile-time constant; no per-call dispatch into `Limits`. - @max_string_bytes Limits.max_string_bytes() - - @compile {:inline, concat_checked: 2} - defp concat_checked(left, right) when byte_size(left) + byte_size(right) <= @max_string_bytes do + # threatens the host. `byte_size/1` is O(1) and the ceiling is a struct + # field read (`state.max_string_bytes`, settable via `Lua.new/1`), so + # this stays cheap on the hot path. + @compile {:inline, concat_checked: 3} + defp concat_checked(left, right, max) when byte_size(left) + byte_size(right) <= max do left <> right end - defp concat_checked(_left, _right) do + defp concat_checked(_left, _right, _max) do raise RuntimeError, value: "resulting string too large" end @@ -2283,7 +2301,7 @@ defmodule Lua.VM.Executor do table = Map.fetch!(state.tables, id) - case Table.get_data(table.data, key) do + case Table.get(table, key) do nil -> case table.metatable do nil -> @@ -2342,7 +2360,7 @@ defmodule Lua.VM.Executor do %{state | tables: Map.put(state.tables, id, updated)} {:tref, mt_id} -> - if Table.has_data?(table.data, key) do + if Table.has_key?(table, key) do updated = Table.put(table, key, value) %{state | tables: Map.put(state.tables, id, updated)} else @@ -2381,7 +2399,7 @@ defmodule Lua.VM.Executor do try_unary_metamethod("__len", tref, state, fn -> {:tref, id} = tref table = Map.fetch!(state.tables, id) - Value.sequence_length(table.data) + Table.length(table) end) case raw do diff --git a/lib/lua/vm/limits.ex b/lib/lua/vm/limits.ex index 7680e02a..2f7e945e 100644 --- a/lib/lua/vm/limits.ex +++ b/lib/lua/vm/limits.ex @@ -42,13 +42,16 @@ defmodule Lua.VM.Limits do @doc """ Asserts that an as-yet-unallocated string of `bytes` bytes is within the - ceiling. Raises a catchable "resulting string too large" runtime error - otherwise. + given ceiling (a state's `max_string_bytes`, defaulting to the practical + bound here). Raises a catchable "resulting string too large" runtime + error otherwise. """ - @spec check_string_size!(integer()) :: :ok - def check_string_size!(bytes) when is_integer(bytes) and bytes <= @max_string_bytes, do: :ok + @spec check_string_size!(integer(), pos_integer()) :: :ok + def check_string_size!(bytes, max \\ @max_string_bytes) - def check_string_size!(_bytes) do + def check_string_size!(bytes, max) when is_integer(bytes) and bytes <= max, do: :ok + + def check_string_size!(_bytes, _max) do raise RuntimeError, value: "resulting string too large" end diff --git a/lib/lua/vm/state.ex b/lib/lua/vm/state.ex index cbd161f0..a7cee14f 100644 --- a/lib/lua/vm/state.ex +++ b/lib/lua/vm/state.ex @@ -3,6 +3,7 @@ defmodule Lua.VM.State do Runtime state for the Lua VM. """ + alias Lua.VM.Limits alias Lua.VM.Table defstruct call_stack: [], @@ -12,6 +13,13 @@ defmodule Lua.VM.State do # means no limit. See `check_call_depth!/1`. call_depth: 0, max_call_depth: :infinity, + # Ceiling for any single string the VM will build (`..`, + # `string.rep`, `load` reader chunks). Defaults to the practical + # bound in `Lua.VM.Limits`. Embedders running the VM under a + # process heap cap (`:max_heap_size`) should set this below that + # cap so an allocation bomb is refused deterministically instead + # of racing the GC-time heap check. + max_string_bytes: Limits.max_string_bytes(), metatables: %{}, upvalue_cells: %{}, open_upvalues: %{}, @@ -31,6 +39,7 @@ defmodule Lua.VM.State do call_stack: list(), call_depth: non_neg_integer(), max_call_depth: pos_integer() | :infinity, + max_string_bytes: pos_integer(), metatables: map(), upvalue_cells: map(), tables: %{optional(non_neg_integer()) => Table.t()}, diff --git a/lib/lua/vm/stdlib.ex b/lib/lua/vm/stdlib.ex index b30baac2..790c6539 100644 --- a/lib/lua/vm/stdlib.ex +++ b/lib/lua/vm/stdlib.ex @@ -260,7 +260,7 @@ defmodule Lua.VM.Stdlib do # rawget(table, key) — get without metamethods defp lua_rawget([{:tref, id}, key | _], state) do table = Map.fetch!(state.tables, id) - {[Table.get_data(table.data, key)], state} + {[Table.get(table, key)], state} end # rawset(table, key, value) — set without metamethods @@ -277,7 +277,7 @@ defmodule Lua.VM.Stdlib do # values, which broke `pcall(rawlen, ...)` patterns in events.lua. defp lua_rawlen([{:tref, id} | _], state) do table = Map.fetch!(state.tables, id) - {[Value.sequence_length(table.data)], state} + {[Table.length(table)], state} end defp lua_rawlen([v | _], state) when is_binary(v) do @@ -483,7 +483,7 @@ defmodule Lua.VM.Stdlib do # pair: an oversized reader is a DoS attempt, not a recoverable # syntax error, so we surface it as a catchable runtime error # (caught by `pcall`) instead of a quiet load failure. - Limits.check_string_size!(size + byte_size(piece)) + Limits.check_string_size!(size + byte_size(piece), state.max_string_bytes) collect_reader_chunks(reader, state, [piece | acc], size + byte_size(piece)) [bad | _] -> diff --git a/lib/lua/vm/stdlib/string.ex b/lib/lua/vm/stdlib/string.ex index 21c7d95c..76d70cd3 100644 --- a/lib/lua/vm/stdlib/string.ex +++ b/lib/lua/vm/stdlib/string.ex @@ -216,7 +216,7 @@ defmodule Lua.VM.Stdlib.String do # Compute the result size from the count *before* building it, so an # oversized request fails with a catchable error instead of trying to # allocate (and OOM the host on) a multi-petabyte binary. - Limits.check_string_size!(n * byte_size(str) + (n - 1) * byte_size(sep)) + Limits.check_string_size!(n * byte_size(str) + (n - 1) * byte_size(sep), state.max_string_bytes) Enum.map_join(1..n, sep, fn _ -> str end) end @@ -794,7 +794,7 @@ defmodule Lua.VM.Stdlib.String do # reads don't mutate state, so we pass it through unchanged. fn [match | _], st -> table = State.get_table(st, repl) - value = Map.get(table.data, match, match) + value = Lua.VM.Table.get(table, match) || match {value, st} end diff --git a/lib/lua/vm/stdlib/table.ex b/lib/lua/vm/stdlib/table.ex index 7f06ac85..a7e1339a 100644 --- a/lib/lua/vm/stdlib/table.ex +++ b/lib/lua/vm/stdlib/table.ex @@ -225,7 +225,7 @@ defmodule Lua.VM.Stdlib.Table do table.metatable == nil -> elements = Enum.map(i..j//1, fn idx -> - concat_value(Table.get_data(table.data, idx), idx) + concat_value(Table.get(table, idx), idx) end) {elements, state} @@ -305,7 +305,7 @@ defmodule Lua.VM.Stdlib.Table do end defp sort_plain(id, table, len, comp, state) do - values = Enum.map(1..len//1, fn i -> Table.get_data(table.data, i) end) + values = Enum.map(1..len//1, fn i -> Table.get(table, i) end) {sorted, state} = sort_values(values, comp, state) @@ -315,15 +315,13 @@ defmodule Lua.VM.Stdlib.Table do # Sort only reorders the values under keys 1..len; every other key # (string fields, sparse integers > len, etc.) must survive untouched, - # matching Lua 5.3 table.sort. Keys 1..len already exist in `data` and - # in `order`, so merging the sorted slice over the existing map changes - # values in place without disturbing order/dead or any other key. - sorted_slice = - sorted - |> Enum.with_index(1) - |> Map.new(fn {val, idx} -> {idx, val} end) + # matching Lua 5.3 table.sort. Keys 1..len occupy the dense array + # border, so writing the sorted slice back through the split-aware + # `put_many/2` rewrites those array slots in place without disturbing + # the hash portion or any other key. + pairs = Enum.with_index(sorted, fn val, idx -> {idx + 1, val} end) - updated = %{table | data: Map.merge(table.data, sorted_slice)} + updated = Table.put_many(table, pairs) {[], %{state | tables: Map.put(state.tables, id, updated)}} end diff --git a/lib/lua/vm/table.ex b/lib/lua/vm/table.ex index 850bdfa2..5a9dacf5 100644 --- a/lib/lua/vm/table.ex +++ b/lib/lua/vm/table.ex @@ -1,46 +1,44 @@ defmodule Lua.VM.Table do @moduledoc """ - Lua table data structure. + Lua table data structure with split array/hash storage. - A single Elixir map backing both array and hash portions, plus a list of - keys in insertion order and a key-set of "dead" keys whose values were - cleared during iteration. + Dense positive-integer keys (`1..n`) live in an Erlang `:array` (the + `arr` field), giving O(1) functional read/write and a free sequence + length. Every other key — strings, non-positive integers, sparse + integers beyond the contiguous array border, float/boolean/table keys — + lives in the `data` hash map alongside the iteration-order bookkeeping. - Keys and values are VM values (numbers, strings, booleans, `{:tref, id}`, - etc.). Integer keys use 1-based indexing per Lua convention. + Keeping string keys in `data` means the executor's field/global fast + paths (`%{^name => value}`) and metatable lookups read the hash map + directly with no change. Integer-keyed reads/writes route through the + split-aware helpers in this module. + + ## Array border + + `arr_n` is the count of contiguous integer keys `1..arr_n` currently + stored in `arr`. Writing key `arr_n + 1` extends the border; clearing a + key inside the border splits it (the tail beyond the hole migrates to + `data` lazily on read). A write to a sparse integer key beyond the + border goes to `data` until a later contiguous fill promotes it. ## Dead-key tracking - Lua 5.3 §6.1 says iteration with `pairs` is well-defined when the body - clears existing fields (`t[k] = nil`). The reference implementation - preserves the iteration sequence by leaving cleared keys reachable in - the hash chain (marked `TDEADKEY`) so the next call to `next(t, k)` can - still find the entry that follows `k`. - - We mirror that behavior with two pieces of state: - - * `order` — keys in the order they were first assigned a value. Live - and dead keys both appear; assigning a fresh value to a previously - dead key moves it to the end (it counts as a new insertion). - * `dead` — a `key => true` map of keys that have been assigned `nil`. - Their slot in `order` is preserved so `next(t, k)` can locate the - slot, but `data` no longer contains the key, so the key is reported - as absent to readers. (We use a plain map rather than `MapSet` here - so dialyzer can treat the empty default opaquely; the API surface - is the same — `Map.has_key?/2`, `Map.put/3`, `Map.delete/2`.) - - All mutations should flow through `put/3` (or `put_data/3` for code that - only has access to the underlying data map and doesn't care about the - iteration ordering). + Lua 5.3 §6.1 dead-key iteration semantics are preserved for the hash + portion via `order`/`order_tail`/`dead` exactly as before. Array-portion + keys iterate first, in index order, then hash keys in insertion order. """ - defstruct data: %{}, + defstruct arr: :undefined, + arr_n: 0, + data: %{}, order: [], order_tail: [], dead: %{}, metatable: nil @type t :: %__MODULE__{ + arr: :array.array() | :undefined, + arr_n: non_neg_integer(), data: %{optional(term()) => term()}, order: list(term()), order_tail: list(term()), @@ -48,69 +46,93 @@ defmodule Lua.VM.Table do metatable: {:tref, non_neg_integer()} | nil } - # `order` is the "stable" iteration list (insertion order, oldest first). - # `order_tail` accumulates **newly inserted** keys in reverse-insertion - # order (newest first). Writes prepend onto `order_tail` in O(1); reads - # via `next_entry/2` flush the tail into `order` lazily so the iteration - # protocol still sees a single ordered list. This avoids the O(n) per - # write that `order ++ [key]` was costing the table_build microbenchmark - # (~25% of its runtime according to tprof). + @compile {:inline, positive_int?: 1} + + defp positive_int?(k) when is_integer(k) and k >= 1, do: true + defp positive_int?(_), do: false + + defp ensure_arr(:undefined), do: :array.new({:default, nil}) + defp ensure_arr(arr), do: arr @doc """ Builds a table struct from a plain data map. - `order` is derived from the data map's key list — Erlang maps surface - their keys in a deterministic order, so callers that pass us a literal - data map (e.g. stdlib initialization) get a sensible iteration order - with no extra effort. + Splits dense positive-integer keys into the array and leaves the rest in + `data`, so callers that pass a literal map (stdlib init, `table.pack`, + encode) get split storage with no extra effort. """ @spec from_data(map()) :: t() def from_data(data) when is_map(data) do - %__MODULE__{data: data, order: Map.keys(data)} + split_from_map(%__MODULE__{}, data) end @doc """ - Replaces the data map wholesale, rebuilding `order` and clearing `dead`. - - Used by stdlib operations that rewrite the entire table contents (e.g. - `table.sort` shuffles every integer key). After this call, iteration - order reflects the new map layout. + Replaces the table contents wholesale from a plain data map, rebuilding + the array/hash split and clearing `dead`. """ @spec replace_data(t(), map()) :: t() def replace_data(%__MODULE__{} = table, data) when is_map(data) do - %{table | data: data, order: Map.keys(data), dead: %{}} + split_from_map(%{table | arr: :undefined, arr_n: 0, data: %{}, order: [], order_tail: [], dead: %{}}, data) + end + + defp split_from_map(table, data) do + Enum.reduce(data, table, fn {k, v}, acc -> put(acc, k, v) end) end @doc """ Writes `value` into the table under `key`, honoring Lua semantics: - * Assigning `nil` removes the key from `data` and marks it dead in - `order` if it was previously live (Lua 5.3 §3.4.11 / §6.1). - * Any other value is stored normally; if the key was previously - dead, it is revived and re-appended to `order` so the new - assignment counts as a fresh insertion. - - Used by every code path that mutates table contents (`set_table`, - `set_field`, `set_list`, `rawset`, `table.insert`, etc.) so the - insertion-order invariant stays consistent. + * Assigning `nil` removes the key. + * Dense positive integers route to the array; other keys to the hash. """ @spec put(t(), term(), term()) :: t() def put(%__MODULE__{} = table, key, value) do key = normalize_key(key) - case value do - nil -> delete(table, key) - _ -> insert(table, key, value) + cond do + value == nil -> delete(table, key) + positive_int?(key) -> put_array(table, key, value) + true -> insert_hash(table, key, value) + end + end + + # Array write. A contiguous append (key == arr_n + 1) extends the border; + # an in-border overwrite is a plain :array.set; a write beyond the border + # goes to the hash until a later contiguous fill could promote it. + defp put_array(%__MODULE__{arr: arr, arr_n: n} = table, key, value) when key == n + 1 do + arr = :array.set(key - 1, value, ensure_arr(arr)) + # Pull any contiguous successors that were parked in the hash into arr. + absorb_from_hash(%{table | arr: arr, arr_n: key}) + end + + defp put_array(%__MODULE__{arr_n: n} = table, key, value) when key <= n do + %{table | arr: :array.set(key - 1, value, table.arr)} + end + + defp put_array(%__MODULE__{} = table, key, value) do + # Sparse integer beyond the border — store in hash for now. + insert_hash(table, key, value) + end + + # After extending the border, migrate any hash-resident keys arr_n+1, + # arr_n+2, ... into the array so the contiguous run stays in arr. + defp absorb_from_hash(%__MODULE__{arr_n: n, data: data} = table) do + next = n + 1 + + case data do + %{^next => v} -> + arr = :array.set(next - 1, v, table.arr) + table = drop_hash_key(%{table | arr: arr, arr_n: next}, next) + absorb_from_hash(table) + + _ -> + table end end - defp insert(%__MODULE__{data: data, order: order, order_tail: order_tail, dead: dead} = table, key, value) do + defp insert_hash(%__MODULE__{data: data, order: order, order_tail: order_tail, dead: dead} = table, key, value) do cond do Map.has_key?(dead, key) -> - # Reviving a dead key — drop it from the existing order/tail and - # re-append so the observable insertion order matches a fresh - # assignment. We have to flush the tail because the dead key could - # live in either list. merged_order = order ++ Enum.reverse(order_tail) new_order = Enum.reject(merged_order, &(&1 === key)) @@ -123,36 +145,52 @@ defmodule Lua.VM.Table do } Map.has_key?(data, key) -> - # Update of an existing live key — value changes, position stable. %{table | data: Map.put(data, key, value)} true -> - # Brand-new key. Prepend onto `order_tail` (O(1)) instead of - # appending to `order` (O(n)). Lazy flush on iteration. %{table | data: Map.put(data, key, value), order_tail: [key | order_tail]} end end + defp delete(%__MODULE__{arr_n: n} = table, key) when is_integer(key) and key >= 1 and key <= n do + # Clear the slot but keep `arr_n` as the high-water bound of the + # allocated array region. Leaving the nil hole (rather than shrinking + # the border) preserves two invariants that iteration relies on: + # + # * `next_entry/2` can still tell a former array key (`key <= arr_n`, + # resume in-array) from a key that was never present (`key > arr_n`, + # raise "invalid key to next") — even after the tail is cleared + # mid-iteration, as Lua 5.3 §6.1 requires. + # * Re-inserting the same key (`t[k] = nil; t[k] = v`) lands back in + # the same array slot, so the dense ordering is stable. + # + # `length/1` derives the Lua border by scanning to the first hole, so a + # cleared tail still reports the correct `#t`. + %{table | arr: :array.set(key - 1, nil, table.arr)} + end + defp delete(%__MODULE__{data: data, dead: dead} = table, key) do if Map.has_key?(data, key) do - # Live key being cleared — move to dead set, leave `order` slot - # in place so any in-flight iteration can still walk past it. %{table | data: Map.delete(data, key), dead: Map.put(dead, key, true)} else - # Already absent (never present, or already cleared) — no-op. - # Per Lua 5.3 §3.4.11, fields with nil values are absent from the - # table, so storing nil over an absent key is a no-op. table end end + defp drop_hash_key(%__MODULE__{data: data, order: order, order_tail: tail} = table, key) do + %{ + table + | data: Map.delete(data, key), + order: Enum.reject(order, &(&1 === key)), + order_tail: Enum.reject(tail, &(&1 === key)) + } + end + @doc """ - Writes `value` into a raw data map under `key`. + Writes `value` into a raw data map under `key` (hash-only, no array). - Lower-level than `put/3`: operates only on the underlying map, with no - awareness of `order`/`dead`. Use this when you have a data map but no - surrounding `Table` struct (e.g. while folding through `set_list` - intermediate state). Prefer `put/3` whenever you have the full struct. + Retained for callers that operate on a bare hash map with no surrounding + struct. Integer keys still land in the map here. """ @spec put_data(map(), term(), term()) :: map() def put_data(data, key, value) do @@ -165,85 +203,25 @@ defmodule Lua.VM.Table do end @doc """ - Applies an ordered list of `{key, value}` writes to the table, producing - the same result as folding `put/3` over the list left-to-right, but - rebuilding the `%Table{}` struct only once at the end. - - This is the batch entry point for table-constructor backfill - (`:set_list`), where a run of consecutive integer keys is written in one - go. The `order`/`order_tail`/`dead` invariants are maintained identically - to repeated `put/3`: new keys append (via `order_tail`), existing live - keys keep their position, dead keys revive to the end, and `nil` values - clear a live key into `dead`. The win is collapsing `count` struct - rebuilds into one. - - `pairs` are applied in order; keys are normalized per `normalize_key/1`. + Applies an ordered list of `{key, value}` writes, equivalent to folding + `put/3` left-to-right but rebuilding the struct once. """ @spec put_many(t(), [{term(), term()}]) :: t() def put_many(%__MODULE__{} = table, []), do: table - def put_many(%__MODULE__{data: data, order: order, order_tail: order_tail, dead: dead} = table, pairs) - when is_list(pairs) do - {data, order, order_tail, dead} = put_many_reduce(pairs, data, order, order_tail, dead) - %{table | data: data, order: order, order_tail: order_tail, dead: dead} - end - - defp put_many_reduce([], data, order, order_tail, dead), do: {data, order, order_tail, dead} - - defp put_many_reduce([{key, value} | rest], data, order, order_tail, dead) do - key = normalize_key(key) - - {data, order, order_tail, dead} = - case value do - nil -> delete_acc(data, order, order_tail, dead, key) - _ -> insert_acc(data, order, order_tail, dead, key, value) - end - - put_many_reduce(rest, data, order, order_tail, dead) - end - - # Accumulator-threaded mirrors of `insert/3` and `delete/2`. They make the - # exact same `order`/`order_tail`/`dead` decisions, but operate on bare - # fields so `put_many/2` can rebuild the struct just once. - defp insert_acc(data, order, order_tail, dead, key, value) do - cond do - Map.has_key?(dead, key) -> - merged_order = order ++ Enum.reverse(order_tail) - new_order = Enum.reject(merged_order, &(&1 === key)) - {Map.put(data, key, value), new_order, [key], Map.delete(dead, key)} - - Map.has_key?(data, key) -> - {Map.put(data, key, value), order, order_tail, dead} - - true -> - {Map.put(data, key, value), order, [key | order_tail], dead} - end - end - - defp delete_acc(data, order, order_tail, dead, key) do - if Map.has_key?(data, key) do - {Map.delete(data, key), order, order_tail, Map.put(dead, key, true)} - else - {data, order, order_tail, dead} - end + def put_many(%__MODULE__{} = table, pairs) when is_list(pairs) do + Enum.reduce(pairs, table, fn {k, v}, acc -> put(acc, k, v) end) end @doc """ Normalizes a table key per Lua 5.3 §3.4.11. - - Float keys that hold an exact integer value are coerced to integers so - that `t[1.0]` and `t[1]` refer to the same slot. NaN keys are left as - floats; callers that disallow NaN keys (`set_table`, `rawset`) detect - the NaN and raise before reaching the data map. """ @spec normalize_key(term()) :: term() def normalize_key(key) when is_float(key) do cond do - # NaN is not equal to itself; leave as-is so the caller can detect it. key != key -> key - # Integer-valued floats convert to integers. key == trunc(key) and key >= -9_223_372_036_854_775_808.0 and key <= 9_223_372_036_854_775_807.0 -> trunc(key) @@ -256,10 +234,6 @@ defmodule Lua.VM.Table do @doc """ Returns true if a key is invalid for use in a table assignment. - - Per Lua 5.3 §3.4.11 / §6.1, table keys cannot be `nil` or `NaN`. - Callers should `raise` with the appropriate "table index is nil" / - "table index is NaN" error message before mutating table data. """ @spec invalid_key?(term()) :: boolean() def invalid_key?(nil), do: true @@ -267,58 +241,168 @@ defmodule Lua.VM.Table do def invalid_key?(_), do: false @doc """ - Reads a value from a table data map, applying Lua key normalization - (integer-valued floats collapse to integers per §3.4.11). + Reads a value from a table by key, split-aware (array then hash). + + This is the struct-level read. Prefer it over the bare-map `get_data/2` + for any code that has the full `%Table{}`. + """ + @spec get(t(), term()) :: term() + def get(%__MODULE__{arr: arr, arr_n: n}, key) when is_integer(key) and key >= 1 and key <= n do + :array.get(key - 1, arr) + end + + def get(%__MODULE__{arr: arr, arr_n: n, data: data}, key) when is_float(key) do + # An integer-valued float (`t[1.0]`) collapses to its integer slot and + # may live in the dense array; everything else stays in the hash. + case normalize_key(key) do + k when is_integer(k) and k >= 1 and k <= n -> :array.get(k - 1, arr) + k -> Map.get(data, k) + end + end + + def get(%__MODULE__{data: data}, key) do + Map.get(data, key) + end + + @doc """ + Reads a value from a bare hash data map, applying key normalization. + + Hash-only: does not consult the array. Callers with a full struct should + use `get/2`. """ @spec get_data(map(), term()) :: term() def get_data(data, key), do: Map.get(data, normalize_key(key)) @doc """ - Returns true when the table data map has an entry for the given key - after normalization. + Returns true when the table (array or hash) has a live entry for `key`. + """ + @spec has_key?(t(), term()) :: boolean() + def has_key?(%__MODULE__{arr: arr, arr_n: n}, key) when is_integer(key) and key >= 1 and key <= n do + :array.get(key - 1, arr) != nil + end + + def has_key?(%__MODULE__{arr: arr, arr_n: n, data: data}, key) when is_float(key) do + case normalize_key(key) do + k when is_integer(k) and k >= 1 and k <= n -> :array.get(k - 1, arr) != nil + k -> Map.has_key?(data, k) + end + end + + def has_key?(%__MODULE__{data: data}, key) do + Map.has_key?(data, key) + end + + @doc """ + Returns true when the bare hash data map has an entry for `key`. """ @spec has_data?(map(), term()) :: boolean() def has_data?(data, key), do: Map.has_key?(data, normalize_key(key)) @doc """ - Returns the next key/value pair in iteration order after `key`. + Returns the Lua sequence length: the largest `n` with keys `1..n` present. + + The contiguous array border gives this for free; we only fall back to + probing the hash for sparse integers parked beyond the border. + """ + @spec length(t()) :: non_neg_integer() + def length(%__MODULE__{arr_n: 0, data: data}), do: probe_hash_length(data, 1, 0) + + def length(%__MODULE__{arr: arr, arr_n: n, data: data}) do + case array_border(arr, n) do + # No hole inside the array region — the border may extend into sparse + # integer keys parked in the hash, so keep probing from arr_n + 1. + ^n -> probe_hash_length(data, n + 1, n) + # A nil hole inside the array region caps the border early. + border -> border + end + end - Walks the table's `order` list to find `key`, then advances through any - dead-key slots until a live entry is found. Returns `{k, v}` for the - next live entry, or `nil` when iteration is complete. + # Returns the largest `b <= n` such that array slots `1..b` are all + # non-nil (i.e. the Lua sequence border within the dense region). A + # full region returns `n`; a leading hole at slot 1 returns 0. + defp array_border(_arr, 0), do: 0 - When `key` is `nil`, returns the first live entry (or `nil` if the - table is empty/all-dead). + defp array_border(arr, n) do + array_border(arr, 1, n) + end - When `key` is non-nil and is not present in `order` at all, returns the - sentinel `:invalid_key` so the caller can raise the user-facing - "invalid key to 'next'" error per Lua 5.3 §6.1. + defp array_border(arr, i, n) when i <= n do + case :array.get(i - 1, arr) do + nil -> i - 1 + _ -> array_border(arr, i + 1, n) + end + end + + defp array_border(_arr, _i, n), do: n + + defp probe_hash_length(data, probe, last) do + if Map.has_key?(data, probe) do + probe_hash_length(data, probe + 1, probe) + else + last + end + end + + @doc """ + Materializes the full table contents (array + hash) as a single flat map. + + Used by code paths that genuinely need the whole table as a map (decode, + display, `string.gsub` replacement lookups). Walks the array once. + """ + @spec to_map(t()) :: map() + def to_map(%__MODULE__{arr: :undefined, data: data}), do: data + + def to_map(%__MODULE__{arr: arr, arr_n: n, data: data}) do + Enum.reduce(1..n//1, data, fn i, acc -> + case :array.get(i - 1, arr) do + nil -> acc + v -> Map.put(acc, i, v) + end + end) + end + + @doc """ + Returns the next key/value pair in iteration order after `key`. + + Array keys (`1..arr_n`) iterate first in index order, then hash keys in + insertion order. Mirrors the old `next_entry/2` contract: returns `{k, v}`, + `nil` at the end, or `:invalid_key` when `key` is absent everywhere. """ @spec next_entry(t(), term()) :: {term(), term()} | nil | :invalid_key def next_entry(%__MODULE__{} = table, nil) do - first_live(merged_order(table), table.data) + case first_array_live(table, 1) do + nil -> first_live(merged_order(table), table.data) + pair -> pair + end end - def next_entry(%__MODULE__{} = table, key) do + def next_entry(%__MODULE__{arr_n: n} = table, key) do key = normalize_key(key) - case advance_past(merged_order(table), key) do - :not_found -> - # The key was never in this table, even as a dead slot. Lua spec - # §6.1 requires raising for this — caller does the raising. - :invalid_key + if is_integer(key) and key >= 1 and key <= n do + case first_array_live(table, key + 1) do + nil -> first_live(merged_order(table), table.data) + pair -> pair + end + else + case advance_past(merged_order(table), key) do + :not_found -> :invalid_key + remaining -> first_live(remaining, table.data) + end + end + end + + defp first_array_live(%__MODULE__{arr_n: n}, i) when i > n, do: nil - remaining -> - first_live(remaining, table.data) + defp first_array_live(%__MODULE__{arr: arr} = table, i) do + case :array.get(i - 1, arr) do + nil -> first_array_live(table, i + 1) + v -> {i, v} end end @doc """ - Flushes any pending appends in `order_tail` into `order`. - - Idempotent: a table with an empty tail is returned unchanged. Used by - callers that want to amortize the cost of repeated `next_entry` calls - (e.g. `lua_next` in the stdlib, which iterates via repeated calls). + Flushes any pending `order_tail` appends into `order`. Idempotent. """ @spec flush_order(t()) :: t() def flush_order(%__MODULE__{order_tail: []} = table), do: table @@ -327,9 +411,6 @@ defmodule Lua.VM.Table do %{table | order: order ++ Enum.reverse(tail), order_tail: []} end - # Internal: produces the conceptual full insertion order without - # mutating the struct. Used by read paths that don't (or can't) write - # back a flushed version. defp merged_order(%__MODULE__{order: order, order_tail: []}), do: order defp merged_order(%__MODULE__{order: order, order_tail: tail}), do: order ++ Enum.reverse(tail) diff --git a/lib/lua/vm/value.ex b/lib/lua/vm/value.ex index 7c490724..df1fc874 100644 --- a/lib/lua/vm/value.ex +++ b/lib/lua/vm/value.ex @@ -272,7 +272,7 @@ defmodule Lua.VM.Value do def decode({:tref, id}, state) do table = Map.fetch!(state.tables, id) - Enum.map(table.data, fn {k, v} -> {k, decode(v, state)} end) + Enum.map(Lua.VM.Table.to_map(table), fn {k, v} -> {k, decode(v, state)} end) end def decode(value, _state), do: value diff --git a/mix.exs b/mix.exs index e8a8d720..83c9fa27 100644 --- a/mix.exs +++ b/mix.exs @@ -82,8 +82,7 @@ defmodule Lua.MixProject do {:stream_data, "~> 1.1", only: [:test]}, {:styler, "~> 1.10", only: [:dev, :test], runtime: false}, {:benchee, "~> 1.3", only: :benchmark}, - {:luerl, "~> 1.5", only: :benchmark}, - {:luaport, "~> 1.6", only: :benchmark} + {:luerl, "~> 1.5", only: :benchmark} ] end end diff --git a/test/lua/vm/limits_test.exs b/test/lua/vm/limits_test.exs index 66c81773..31a087c5 100644 --- a/test/lua/vm/limits_test.exs +++ b/test/lua/vm/limits_test.exs @@ -40,6 +40,74 @@ defmodule Lua.VM.LimitsTest do end end + describe ":max_string_bytes option" do + # A 1 KB ceiling keeps these tests allocation-free while exercising + # every guard site at a size far below the default 256 MiB bound — + # the behavior an embedder relies on when running the VM under a + # process heap cap. + setup do + %{small: Lua.new(sandboxed: [], max_string_bytes: 1024)} + end + + test "string.rep honors a lowered ceiling", %{small: small} do + assert pcall_error(small, ~s|return string.rep("x", 2048)|) =~ + "resulting string too large" + + assert {["xxxx"], _} = Lua.eval!(small, ~s|return string.rep("x", 4)|) + end + + test "concatenation honors a lowered ceiling on both execution paths", %{small: small} do + # Doubling from inside a function body exercises the compiled + # dispatcher; the top-level loop exercises the interpreter. + doubling = "local s = 'x' for _ = 1, 11 do s = s .. s end return s" + assert pcall_error(small, doubling) =~ "resulting string too large" + + in_function = """ + local function grow() + local s = "x" + for _ = 1, 11 do s = s .. s end + return s + end + local ok, err = pcall(grow) + return ok, tostring(err) + """ + + assert {[false, message], _} = Lua.eval!(small, in_function) + assert message =~ "resulting string too large" + + assert {["ab"], _} = Lua.eval!(small, ~s|return "a" .. "b"|) + end + + test "load reader chunks honor a lowered ceiling", %{small: small} do + code = """ + local piece = string.rep("-", 512) + local n = 0 + local ok, err = pcall(load, function() + n = n + 1 + if n > 8 then return "" end + return piece + end) + return ok, tostring(err) + """ + + assert {[false, message], _} = Lua.eval!(small, code) + assert message =~ "resulting string too large" + end + + test "the default ceiling is unchanged", %{lua: lua} do + # 1 MB is far under the 256 MiB default; must build fine. + assert {[1_048_576], _} = Lua.eval!(lua, ~s|return #string.rep("x", 2^20)|) + end + + test "rejects non-positive and non-integer values" do + for bad <- [0, -1, :infinity, "16m", 1.5] do + assert_raise ArgumentError, ~r/:max_string_bytes must be a positive integer/, fn -> + Lua.new(max_string_bytes: bad) + end + end + end + end + describe "string.format" do test "refuses an oversized precision field", %{lua: lua} do assert pcall_error(lua, ~s|return string.format("%.2000000000f", 1.0)|) =~ diff --git a/test/lua/vm/stdlib/table_test.exs b/test/lua/vm/stdlib/table_test.exs index a9b3317a..5012a3f0 100644 --- a/test/lua/vm/stdlib/table_test.exs +++ b/test/lua/vm/stdlib/table_test.exs @@ -264,10 +264,12 @@ defmodule Lua.VM.Stdlib.TableTest do assert [3, 1, 2, 3] = run!(code) end - test "clearing a constructor key to nil is a hole, not a reorder" do - # Backfill five keys, clear one in the middle, then re-add it. The - # revived key counts as a fresh insertion and moves to the end of - # iteration order — identical to the per-`put` loop's dead-key revival. + test "clearing a dense integer key to nil and re-adding keeps its array slot" do + # Backfill five dense integer keys, clear one in the middle, then + # re-add it. Dense positive integers live in the array, so clearing a + # slot leaves a hole and re-adding fills the same slot — iteration + # stays in index order. This matches reference Lua 5.3, where these + # keys all live in the array part and a revived key keeps its index. code = """ local t = {1, 2, 3, 4, 5} t[3] = nil @@ -279,7 +281,7 @@ defmodule Lua.VM.Stdlib.TableTest do return table.concat(keys, ",") """ - assert ["1,2,4,5,3"] = run!(code) + assert ["1,2,3,4,5"] = run!(code) end test "an overwritten constructor slot keeps its original position" do diff --git a/test/lua/vm/value_test.exs b/test/lua/vm/value_test.exs index 81a244cd..78c1312a 100644 --- a/test/lua/vm/value_test.exs +++ b/test/lua/vm/value_test.exs @@ -2,6 +2,7 @@ defmodule Lua.VM.ValueTest do use ExUnit.Case, async: true alias Lua.VM.State + alias Lua.VM.Table alias Lua.VM.Value defp new_state, do: State.new() @@ -101,9 +102,9 @@ defmodule Lua.VM.ValueTest do {{:tref, id}, state} = Value.encode(["a", "b", "c"], new_state()) table = Map.fetch!(state.tables, id) - assert table.data[1] == "a" - assert table.data[2] == "b" - assert table.data[3] == "c" + assert Table.get(table, 1) == "a" + assert Table.get(table, 2) == "b" + assert Table.get(table, 3) == "c" end test "encodes empty list" do @@ -125,7 +126,7 @@ defmodule Lua.VM.ValueTest do {{:tref, id}, state} = Value.encode([%{x: 1}], new_state()) table = Map.fetch!(state.tables, id) - {:tref, inner_id} = table.data[1] + {:tref, inner_id} = Table.get(table, 1) inner_table = Map.fetch!(state.tables, inner_id) assert inner_table.data["x"] == 1 end diff --git a/website/lib/website/lua_sandbox.ex b/website/lib/website/lua_sandbox.ex index 8b5f6c22..99e25bf8 100644 --- a/website/lib/website/lua_sandbox.ex +++ b/website/lib/website/lua_sandbox.ex @@ -15,6 +15,14 @@ defmodule Website.LuaSandbox do applies a safety wall-clock timeout so a non-terminating snippet always returns; the LiveView layers a shorter UI timeout on top (it wraps `run/1` in `start_async/3` and cancels it on a timer). + + The VM's own string ceiling (`:max_string_bytes`) is set well below the + worker's heap cap. Order matters: the VM refuses an oversized string + *deterministically* (the size is computed before allocating), while + `max_heap_size` only kills when a garbage collection happens to observe + the oversized live set. With the VM ceiling under the heap cap, string + bombs always fail with a catchable Lua error; the heap cap remains the + backstop for what the VM cannot pre-compute (e.g. unbounded tables). """ import Lua, only: [sigil_LUA: 2] @@ -28,6 +36,12 @@ defmodule Website.LuaSandbox do # snippets here while still stopping an accidental infinite recursion. @max_call_depth 200 + # VM string ceiling, deliberately far below @lua_heap_words (64 MB): + # a doubling loop peaks at ~1.5x the ceiling in live binaries, so 16 MB + # keeps the worst case (~24 MB) comfortably inside the heap cap and the + # refusal deterministic. See the moduledoc for why the order matters. + @lua_max_string_bytes 16 * 1024 * 1024 + @doc """ Compiles a Lua snippet into a `Lua.Chunk` (without running it) and returns the chunk plus the disassembled prototype tree. Returns @@ -100,6 +114,19 @@ defmodule Website.LuaSandbox do try do receive do {:eval_result, result} -> + # The worker has delivered its result but its exit-time GC can + # still trip the heap kill (include_shared_binaries counts the + # big binaries it is about to drop). Unlink — and flush an exit + # signal that may already be queued — so a late `:killed` can't + # propagate to us once we stop trapping. + Process.unlink(worker) + + receive do + {:EXIT, ^worker, _} -> :ok + after + 0 -> :ok + end + result # Worker hit the memory ceiling: max_heap_size kills it with @@ -163,7 +190,7 @@ defmodule Website.LuaSandbox do defp eval(source, output_pid, started) do lua = - Lua.new(max_call_depth: @max_call_depth) + Lua.new(max_call_depth: @max_call_depth, max_string_bytes: @lua_max_string_bytes) |> Lua.set!([:print], fn args -> line = args