Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions benchmarks/array_vs_map_probe.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Probe: Erlang :array vs Elixir map for the access patterns the table
# workloads exercise. This isolates the data-structure lever from the rest
# of the VM so we can decide whether split storage can plausibly close the
# build/sort gap vs luerl before paying the full refactor cost.

Code.require_file("helpers.exs", __DIR__)

ns = [100, 1000]

build_map = fn n ->
Enum.reduce(1..n, %{}, fn i, acc -> Map.put(acc, i, i * i) end)
end

build_arr = fn n ->
Enum.reduce(1..n, :array.new([{:size, n}, {:fixed, false}, {:default, nil}]), fn i, acc ->
:array.set(i - 1, i * i, acc)
end)
end

# Pre-build for read/length/sortwrite probes
maps = Map.new(ns, fn n -> {n, build_map.(n)} end)
arrs = Map.new(ns, fn n -> {n, build_arr.(n)} end)

sum_map = fn m, n -> Enum.reduce(1..n, 0, fn i, s -> s + Map.get(m, i) end) end
sum_arr = fn a, n -> Enum.reduce(1..n, 0, fn i, s -> s + :array.get(i - 1, a) end) end

# sort write-back: take sorted values list, write each back under key idx
sortwrite_map = fn n ->
vals = Enum.to_list(n..1//-1)
Enum.reduce(Enum.with_index(vals, 1), %{}, fn {v, idx}, acc -> Map.put(acc, idx, v) end)
end

sortwrite_arr = fn n ->
vals = Enum.to_list(n..1//-1)
base = :array.new([{:size, n}, {:fixed, false}, {:default, nil}])
Enum.reduce(Enum.with_index(vals, 1), base, fn {v, idx}, acc -> :array.set(idx - 1, v, acc) end)
end

# sequence_length: map probes 1..n+1; array is just :array.size
seqlen_map = fn m ->
Stream.iterate(1, &(&1 + 1)) |> Enum.find(fn i -> not Map.has_key?(m, i) end) |> Kernel.-(1)
end

for n <- ns do
Bench.banner("DS probe n=#{n}")

Benchee.run(
%{
"build map" => fn -> build_map.(n) end,
"build array" => fn -> build_arr.(n) end
},
Bench.opts()
)

Benchee.run(
%{
"sum map (read)" => fn -> sum_map.(maps[n], n) end,
"sum array (read)" => fn -> sum_arr.(arrs[n], n) end
},
Bench.opts()
)

Benchee.run(
%{
"sortwrite map" => fn -> sortwrite_map.(n) end,
"sortwrite array" => fn -> sortwrite_arr.(n) end
},
Bench.opts()
)

Benchee.run(
%{
"seqlen map (probe)" => fn -> seqlen_map.(maps[n]) end,
"seqlen array (size)" => fn -> :array.size(arrs[n]) end
},
Bench.opts()
)
end
20 changes: 18 additions & 2 deletions guides/sandboxing.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,11 @@ These guards are always on and need no configuration. They cover:

| Operation | Guard | Error (catchable with `pcall`) |
| :-------- | :---- | :----------------------------- |
| `string.rep`, the `..` operator | result larger than ~256 MiB | `resulting string too large` |
| `string.rep`, the `..` operator | result larger than the string ceiling (default ~256 MiB) | `resulting string too large` |
| `string.format` width/precision | field wider than 99 | `invalid conversion` |
| `table.unpack` | more than 10M results | `too many results to unpack` |
| `table.concat`, `table.move` | range wider than 10M | `range too large` |
| `load` (when enabled) | reader returns > ~256 MiB total | `resulting string too large` |
| `load` (when enabled) | reader returns more than the string ceiling in total | `resulting string too large` |

```elixir
{[false, message], _} =
Expand All @@ -111,6 +111,22 @@ message =~ "resulting string too large"
Because these surface as ordinary Lua errors, a script can even recover
from them with `pcall`.

The string ceiling is configurable via `:max_string_bytes`:

```elixir
lua = Lua.new(max_string_bytes: 16 * 1024 * 1024)
```

> #### Pair the ceiling with your heap cap {: .warning}
> If you run the VM inside a process capped with `:max_heap_size` (see
> below), set `:max_string_bytes` comfortably *below* that cap. The
> default ceiling permits strings large enough that a smaller heap cap
> would have to catch them instead — and `max_heap_size` only kills when
> a garbage collection happens to observe the oversized live set, which
> makes the outcome timing-dependent. With the ceiling under the cap,
> string bombs are always refused deterministically and the heap cap
> stays what it should be: a backstop for what the VM cannot pre-compute.

## Call depth

By default the VM places no limit on call depth, so deeply recursive or
Expand Down
39 changes: 37 additions & 2 deletions lib/lua.ex
Original file line number Diff line number Diff line change
Expand Up @@ -100,15 +100,43 @@ defmodule Lua do
iex> {[false, message], _lua} = Lua.eval!(lua, "local function f() return f() end return pcall(f)")
iex> message =~ "stack overflow"
true

* `:max_string_bytes` - (default 256 MiB) ceiling for any single string the VM will build,
whether via `..`, `string.rep`, or a `load` reader. An oversized result raises a catchable
`"resulting string too large"` error — the size is computed *before* allocating, so the
bomb is refused rather than detected after the fact. Accepts a positive integer.

When running the VM inside a process capped with `:max_heap_size`, set this comfortably
below the heap cap: the default ceiling permits strings large enough to trip a smaller
heap cap mid-allocation, where the kill depends on garbage-collection timing rather than
a deterministic refusal.

iex> lua = Lua.new(max_string_bytes: 1024)
iex> {[false, message], _lua} = Lua.eval!(lua, "return pcall(string.rep, \\"x\\", 2048)")
iex> message =~ "resulting string too large"
true
"""
@spec new(keyword()) :: t()
def new(opts \\ []) do
opts = Keyword.validate!(opts, sandboxed: @default_sandbox, exclude: [], debug: false, max_call_depth: :infinity)
opts =
Keyword.validate!(opts,
sandboxed: @default_sandbox,
exclude: [],
debug: false,
max_call_depth: :infinity,
max_string_bytes: Lua.VM.Limits.max_string_bytes()
)

exclude = Keyword.fetch!(opts, :exclude)
debug = Keyword.fetch!(opts, :debug)
max_call_depth = validate_max_call_depth!(Keyword.fetch!(opts, :max_call_depth))
max_string_bytes = validate_max_string_bytes!(Keyword.fetch!(opts, :max_string_bytes))

state = %{Lua.VM.Stdlib.install(State.new()) | max_call_depth: max_call_depth}
state = %{
Lua.VM.Stdlib.install(State.new())
| max_call_depth: max_call_depth,
max_string_bytes: max_string_bytes
}

opts
|> Keyword.fetch!(:sandboxed)
Expand All @@ -124,6 +152,13 @@ defmodule Lua do
":max_call_depth must be a positive integer or :infinity, got: #{inspect(other)}"
end

defp validate_max_string_bytes!(bytes) when is_integer(bytes) and bytes > 0, do: bytes

defp validate_max_string_bytes!(other) do
raise ArgumentError,
":max_string_bytes must be a positive integer, got: #{inspect(other)}"
end

@doc """
Write Lua code that is parsed at compile-time.

Expand Down
30 changes: 25 additions & 5 deletions lib/lua/vm/dispatcher.ex
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,14 @@ defmodule Lua.VM.Dispatcher do
alias Lua.Compiler.Prototype
alias Lua.VM.Executor
alias Lua.VM.InternalError
alias Lua.VM.Limits
alias Lua.VM.Numeric
alias Lua.VM.RuntimeError
alias Lua.VM.State
alias Lua.VM.Table
alias Lua.VM.Value

# Mirror of the interpreter's concat ceiling, inlined as a compile-time
# constant so the binary-binary fast path stays a single comparison. See
# `Lua.VM.Executor.concat_checked/2` for the rationale.
@max_string_bytes Limits.max_string_bytes()

# Opcode tags. These must stay in lockstep with `Lua.Compiler.Bytecode`.
# The module-attribute form lets each case branch match a constant
Expand Down Expand Up @@ -720,6 +717,29 @@ defmodule Lua.VM.Dispatcher do
key = :erlang.element(key_reg + 1, regs)

case table_val do
{:tref, id} when is_integer(key) and key >= 1 ->
table = :erlang.map_get(id, state.tables)

case Table.get(table, key) do
nil ->
case :erlang.map_get(:metatable, table) do
nil ->
regs = :erlang.setelement(dest + 1, regs, nil)
dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames)

_ ->
{value, state} =
Executor.dispatcher_get_table(table_val, key, state, proto, name_hint)

regs = :erlang.setelement(dest + 1, regs, value)
dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames)
end

value ->
regs = :erlang.setelement(dest + 1, regs, value)
dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames)
end

{:tref, id} when is_integer(key) or is_binary(key) ->
table = :erlang.map_get(id, state.tables)
data = :erlang.map_get(:data, table)
Expand Down Expand Up @@ -789,7 +809,7 @@ defmodule Lua.VM.Dispatcher do
nil ->
# No __len possible — Lua 5.3 §3.4.7: # on a table
# without __len is the border length of the data map.
len = Value.sequence_length(:erlang.map_get(:data, table))
len = Table.length(table)
regs = :erlang.setelement(dest + 1, regs, len)
dispatch(code, pc + 1, regs, upvalues, proto, state, cont, frames)

Expand Down Expand Up @@ -1103,7 +1123,7 @@ defmodule Lua.VM.Dispatcher do
right = :erlang.element(b + 1, regs)

if is_binary(left) and is_binary(right) do
if byte_size(left) + byte_size(right) > @max_string_bytes do
if byte_size(left) + byte_size(right) > state.max_string_bytes do
raise RuntimeError, value: "resulting string too large"
end

Expand Down
2 changes: 1 addition & 1 deletion lib/lua/vm/display.ex
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ defmodule Lua.VM.Display do
defp peek_table(state, id, decode?) do
case Map.fetch(state.tables, id) do
{:ok, table} ->
data = table.data
data = Lua.VM.Table.to_map(table)

if sequence_like?(data) do
Enum.map(1..map_size(data), &wrap_value(Map.fetch!(data, &1), state, decode?))
Expand Down
58 changes: 38 additions & 20 deletions lib/lua/vm/executor.ex
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ defmodule Lua.VM.Executor do

alias Lua.VM.Dispatcher
alias Lua.VM.InternalError
alias Lua.VM.Limits
alias Lua.VM.Numeric
alias Lua.VM.RuntimeError
alias Lua.VM.State
Expand Down Expand Up @@ -356,7 +355,7 @@ defmodule Lua.VM.Executor do
case value do
{:tref, id} ->
table = Map.fetch!(state.tables, id)
Value.sequence_length(table.data)
Table.length(table)

v when is_binary(v) ->
byte_size(v)
Expand Down Expand Up @@ -420,7 +419,7 @@ defmodule Lua.VM.Executor do
src = proto.source

try_binary_metamethod("__concat", left, right, state, fn ->
concat_checked(concat_coerce(left, 0, src), concat_coerce(right, 0, src))
concat_checked(concat_coerce(left, 0, src), concat_coerce(right, 0, src), state.max_string_bytes)
end)
end

Expand Down Expand Up @@ -1351,12 +1350,12 @@ defmodule Lua.VM.Executor do
# also concatenate without metamethods.
cond do
is_binary(left) and is_binary(right) ->
regs = put_elem(regs, dest, concat_checked(left, right))
regs = put_elem(regs, dest, concat_checked(left, right, state.max_string_bytes))
do_execute(rest, regs, upvalues, proto, state, cont, frames, line)

(is_binary(left) or is_number(left)) and (is_binary(right) or is_number(right)) ->
src = proto.source
result = concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src))
result = concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src), state.max_string_bytes)
regs = put_elem(regs, dest, result)
do_execute(rest, regs, upvalues, proto, state, cont, frames, line)

Expand All @@ -1365,7 +1364,7 @@ defmodule Lua.VM.Executor do

{result, new_state} =
try_binary_metamethod("__concat", left, right, state, fn ->
concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src))
concat_checked(concat_coerce(left, line, src), concat_coerce(right, line, src), state.max_string_bytes)
end)

regs = put_elem(regs, dest, result)
Expand Down Expand Up @@ -1627,7 +1626,7 @@ defmodule Lua.VM.Executor do
case value do
{:tref, id} ->
table = Map.fetch!(state.tables, id)
Value.sequence_length(table.data)
Table.length(table)

v when is_binary(v) ->
byte_size(v)
Expand Down Expand Up @@ -1668,10 +1667,30 @@ defmodule Lua.VM.Executor do
key = elem(regs, key_reg)

case table_val do
{:tref, id} when is_integer(key) and key >= 1 ->
# Split-storage fast path: dense positive integers live in the array.
table = :erlang.map_get(id, state.tables)

case Table.get(table, key) do
nil ->
case :erlang.map_get(:metatable, table) do
nil ->
regs = put_elem(regs, dest, nil)
do_execute(rest, regs, upvalues, proto, state, cont, frames, line)

_ ->
{value, state} = index_value(table_val, key, state, line, proto.source, name_hint)
regs = put_elem(regs, dest, value)
do_execute(rest, regs, upvalues, proto, state, cont, frames, line)
end

value ->
regs = put_elem(regs, dest, value)
do_execute(rest, regs, upvalues, proto, state, cont, frames, line)
end

{:tref, id} when is_integer(key) or is_binary(key) ->
# Fast path mirroring get_field: integer/string key on a tref. Skip
# normalize_key (no-op for these) and the full index_value pipeline
# when the entry is present or no metatable is set.
# Non-positive integer / string key: read straight from the hash map.
table = :erlang.map_get(id, state.tables)

case :erlang.map_get(:data, table) do
Expand Down Expand Up @@ -2162,16 +2181,15 @@ defmodule Lua.VM.Executor do
# in one BIF call — faster than the GC-time `max_heap_size` check can
# react. Guard the size deterministically here, matching the Layer A
# stdlib checks, so the loop fails with a catchable error long before it
# threatens the host. `byte_size/1` is O(1), so this is cheap on the hot
# path. Compile-time constant; no per-call dispatch into `Limits`.
@max_string_bytes Limits.max_string_bytes()

@compile {:inline, concat_checked: 2}
defp concat_checked(left, right) when byte_size(left) + byte_size(right) <= @max_string_bytes do
# threatens the host. `byte_size/1` is O(1) and the ceiling is a struct
# field read (`state.max_string_bytes`, settable via `Lua.new/1`), so
# this stays cheap on the hot path.
@compile {:inline, concat_checked: 3}
defp concat_checked(left, right, max) when byte_size(left) + byte_size(right) <= max do
left <> right
end

defp concat_checked(_left, _right) do
defp concat_checked(_left, _right, _max) do
raise RuntimeError, value: "resulting string too large"
end

Expand Down Expand Up @@ -2283,7 +2301,7 @@ defmodule Lua.VM.Executor do

table = Map.fetch!(state.tables, id)

case Table.get_data(table.data, key) do
case Table.get(table, key) do
nil ->
case table.metatable do
nil ->
Expand Down Expand Up @@ -2342,7 +2360,7 @@ defmodule Lua.VM.Executor do
%{state | tables: Map.put(state.tables, id, updated)}

{:tref, mt_id} ->
if Table.has_data?(table.data, key) do
if Table.has_key?(table, key) do
updated = Table.put(table, key, value)
%{state | tables: Map.put(state.tables, id, updated)}
else
Expand Down Expand Up @@ -2381,7 +2399,7 @@ defmodule Lua.VM.Executor do
try_unary_metamethod("__len", tref, state, fn ->
{:tref, id} = tref
table = Map.fetch!(state.tables, id)
Value.sequence_length(table.data)
Table.length(table)
end)

case raw do
Expand Down
Loading
Loading