diff --git a/CHANGELOG.md b/CHANGELOG.md index 7def79e..a8432ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,18 @@ Everything else — the default sandbox, `_G`/`_ENV` semantics, metatables, and the standard-library surface — is compatible. The full breaking-change list is in the [`1.0.0-rc.0`](#100-rc0---2026-05-26) entry below. +## [Unreleased] + +### Changed +- `Lua.new/1` is ~100x faster (roughly 40µs down to 0.4µs) for the default and + fully-custom-sandbox configurations. Installing the standard library is pure + and deterministic, so the boot-time VM template is now built once per node + and memoized in `:persistent_term`; every later `Lua.new/1` starts from the + shared template copy-on-write. In `:interactive` mode (dev, IEx, tests) the + cache self-invalidates when the modules that built it are recompiled; hosts + that hot-load new code in `:embedded` mode (releases) can force a rebuild + with `Lua.VM.Bootstrap.reset/0` (#398). + ## [1.0.1] - 2026-07-16 ### Fixed diff --git a/lib/lua.ex b/lib/lua.ex index 0b47bcd..e055366 100644 --- a/lib/lua.ex +++ b/lib/lua.ex @@ -8,6 +8,7 @@ defmodule Lua do alias Lua.Util alias Lua.VM.AssertionError + alias Lua.VM.Bootstrap alias Lua.VM.Display alias Lua.VM.Executor alias Lua.VM.InternalError @@ -142,23 +143,51 @@ defmodule Lua do max_instructions: :infinity ) + sandboxed = Keyword.fetch!(opts, :sandboxed) exclude = Keyword.fetch!(opts, :exclude) debug = Keyword.fetch!(opts, :debug) max_call_depth = validate_max_call_depth!(Keyword.fetch!(opts, :max_call_depth)) max_string_bytes = validate_max_string_bytes!(Keyword.fetch!(opts, :max_string_bytes)) max_instructions = validate_max_instructions!(Keyword.fetch!(opts, :max_instructions)) - state = %{ - Lua.VM.Stdlib.install(State.new()) - | max_call_depth: max_call_depth, - max_string_bytes: max_string_bytes, - max_instructions: max_instructions + lua = template(sandboxed, exclude) + + %{ + lua + | debug: debug, + state: %{ + lua.state + | max_call_depth: max_call_depth, + max_string_bytes: max_string_bytes, + max_instructions: max_instructions + } } + end + + # A freshly built VM is a pure function of the sandbox options — the limits + # and `:debug` are patched onto the finished struct above, and every later + # mutation is copy-on-write — so the two shapes that dominate real use are + # memoized rather than rebuilt. Installing the standard library was four + # fifths of what `new/1` cost, and it produces the same value every time. + # + # The default arguments hit the fully-sandboxed template. Anything else pays + # only for its own sandbox pass, over the shared pre-sandbox install. + defp template(@default_sandbox, []) do + Bootstrap.fetch({__MODULE__, :default_lua}, fn -> sandbox_all(@default_sandbox, []) end) + end - opts - |> Keyword.fetch!(:sandboxed) + defp template(sandboxed, exclude), do: sandbox_all(sandboxed, exclude) + + defp sandbox_all(sandboxed, exclude) do + lua = %__MODULE__{state: base_state()} + + sandboxed |> Enum.reject(fn path -> path in exclude end) - |> Enum.reduce(%__MODULE__{state: state, debug: debug}, &sandbox(&2, &1)) + |> Enum.reduce(lua, &sandbox(&2, &1)) + end + + defp base_state do + Bootstrap.fetch({__MODULE__, :base_state}, fn -> Lua.VM.Stdlib.install(State.new()) end) end defp validate_max_call_depth!(:infinity), do: :infinity diff --git a/lib/lua/vm/bootstrap.ex b/lib/lua/vm/bootstrap.ex new file mode 100644 index 0000000..27bceb2 --- /dev/null +++ b/lib/lua/vm/bootstrap.ex @@ -0,0 +1,156 @@ +defmodule Lua.VM.Bootstrap do + @moduledoc """ + Memoizes boot-time VM templates in `:persistent_term`. + + Building a VM means installing the whole standard library — allocating a + dozen tables and capturing a hundred-odd native-function closures — and the + result is a pure, deterministic value: two builds from the same code produce + byte-identical terms. Nothing in it is per-instance (no refs, no pids, no + timestamps), and every downstream mutation is copy-on-write, so a single + template can seed unlimited independent VMs. + + `fetch/2` builds a template on first use and stores it under `key`. Later + calls return the stored term for the cost of one `:persistent_term.get/2`. + There is no mutual exclusion: callers racing a cold start each build and + `put`, and every `put` over a live key forces a global scan of every + process. Builders are pure, so the racing puts all store equal terms — the + race costs redundant work on first use, never an inconsistent result — and + once warm the key is effectively write-once, refreshed only by the + staleness path below. + + ## Staleness + + A template holds closures pointing into the modules that built it. Reload + those modules twice and the old code is purged, turning every captured + closure into a `badfun`. That only happens while iterating in a shell, so the + guard is priced for it: at build time, in `:interactive` mode, every module + reachable through a captured fun — plus the template-shaping modules whose + struct layouts and defaults are baked into the term without leaving a + closure in it — is fingerprinted by its `module_info(:md5)`, and a `fetch/2` + hit re-checks those hashes before handing the term back. A mismatch (or a + fingerprinted module that can no longer be loaded) rebuilds. + + Under `:embedded` mode (releases) code never reloads on its own, so no + fingerprint is taken and a hit is the bare `get`. A host that loads new + Lua-implementation code anyway — `:code.load_file/1`, a remote-console + recompile, a hot upgrade — must call `reset/0` afterwards, or `fetch/2` + keeps serving templates built against the replaced code. + """ + + @typep fingerprint :: :static | [{module(), binary() | nil}] + + # Modules whose code shapes a built template without necessarily leaving a + # captured closure inside it: struct layouts, table splitting, and default + # limits are baked into the stored term at build time, so reloading any of + # these must invalidate the template even though no fun in it points there. + @template_modules [__MODULE__, Lua, Lua.VM.Limits, Lua.VM.State, Lua.VM.Stdlib, Lua.VM.Table] + + # Every key `fetch/2` may be called with, and therefore everything `reset/0` + # has to erase. Fixed at compile time on purpose: the store stays a known, + # bounded size, and `reset/0` needs no bookkeeping that a concurrent cold + # start could lose. + @memoized_keys [{Lua, :default_lua}, {Lua, :base_state}] + + # Earlier builds tracked the keys here at runtime. Erased by `reset/0` so a + # release upgraded from one of those builds does not keep the entry forever. + @obsolete_registry {__MODULE__, :keys} + + @doc """ + Returns the memoized template for `key`, building it with `builder` on a miss. + + `key` must be one of the compile-time literals in `memoized_keys/0`. That is + what bounds the store: `:persistent_term` has no eviction, so a key derived + from user input would grow it without limit, and `reset/0` would not know to + erase it. + + `builder` must be pure: it is invoked on a cold start (possibly more than + once under concurrent first use, see the moduledoc) and again after a module + reload in `:interactive` mode; every caller shares the stored result. + """ + @spec fetch(term(), (-> term())) :: term() + def fetch(key, builder) when is_function(builder, 0) do + case :persistent_term.get(key, nil) do + nil -> + build_and_store(key, builder) + + {fingerprint, term} -> + if current?(fingerprint), do: term, else: build_and_store(key, builder) + end + end + + @doc """ + The keys `fetch/2` accepts, and the exact set `reset/0` erases. + """ + @spec memoized_keys() :: [term()] + def memoized_keys, do: @memoized_keys + + @doc """ + Erases every memoized template, so the next `fetch/2` of each key rebuilds. + + Required after explicitly loading new Lua-implementation code in `:embedded` + mode (releases), where no staleness fingerprint exists — see the moduledoc. + Safe to call at any time in any mode; concurrent `fetch/2` callers simply + rebuild. + """ + @spec reset() :: :ok + def reset do + Enum.each([@obsolete_registry | @memoized_keys], &:persistent_term.erase/1) + :ok + end + + defp build_and_store(key, builder) do + term = builder.() + :persistent_term.put(key, {fingerprint(term), term}) + term + end + + defp current?(:static), do: true + + defp current?(fingerprint) do + Enum.all?(fingerprint, fn {module, md5} -> loaded_md5(module) === md5 end) + end + + # A fingerprinted module that has since been deleted has no md5 to compare; + # returning nil makes the comparison fail, so the caller rebuilds instead of + # raising `UndefinedFunctionError` mid-fetch. + defp loaded_md5(module) do + case :code.ensure_loaded(module) do + {:module, ^module} -> module.module_info(:md5) + {:error, _reason} -> nil + end + end + + @spec fingerprint(term()) :: fingerprint() + defp fingerprint(term) do + case :code.get_mode() do + :interactive -> + term + |> fun_modules() + |> Enum.concat(@template_modules) + |> Enum.uniq() + |> Enum.map(fn module -> {module, loaded_md5(module)} end) + + _embedded -> + :static + end + end + + # Every module reachable through a closure stored anywhere in the term. Runs + # once per build, so a plain structural walk is fine. + defp fun_modules(fun) when is_function(fun) do + {:module, module} = :erlang.fun_info(fun, :module) + [module] + end + + defp fun_modules(list) when is_list(list), do: Enum.flat_map(list, &fun_modules/1) + + # `:maps.to_list/1` rather than `Enum`: structs are maps here too, and they + # are not enumerable. + defp fun_modules(map) when is_map(map), do: map |> :maps.to_list() |> fun_modules() + + defp fun_modules(tuple) when is_tuple(tuple) do + tuple |> Tuple.to_list() |> fun_modules() + end + + defp fun_modules(_other), do: [] +end diff --git a/lib/lua/vm/state.ex b/lib/lua/vm/state.ex index 248437f..15c3aad 100644 --- a/lib/lua/vm/state.ex +++ b/lib/lua/vm/state.ex @@ -212,6 +212,19 @@ defmodule Lua.VM.State do update_table(state, g_ref, fn table -> Table.put(table, name, value) end) end + @doc """ + Sets a batch of global variables, left to right, in one `_G` update. + + Equivalent to folding `set_global/3` over `pairs`, but the surrounding + `%State{}` and its `tables` map are rebuilt once instead of once per name — + which is what installing the standard library's several dozen globals used + to cost. + """ + @spec set_globals(t(), [{binary(), term()}]) :: t() + def set_globals(%__MODULE__{g_ref: g_ref} = state, pairs) when is_list(pairs) and not is_nil(g_ref) do + update_table(state, g_ref, fn table -> Table.put_many(table, pairs) end) + end + @doc """ Reads a global variable from the VM state. Returns `nil` if unset. """ diff --git a/lib/lua/vm/stdlib.ex b/lib/lua/vm/stdlib.ex index a1caca2..6967692 100644 --- a/lib/lua/vm/stdlib.ex +++ b/lib/lua/vm/stdlib.ex @@ -18,54 +18,74 @@ defmodule Lua.VM.Stdlib do alias Lua.VM.TypeError alias Lua.VM.Value + # Installed in this order: it is the order `pairs(_G)` and + # `pairs(package.loaded)` report the library tables in. + @libraries [ + Lua.VM.Stdlib.String, + Lua.VM.Stdlib.Math, + Lua.VM.Stdlib.Table, + Lua.VM.Stdlib.Utf8, + Lua.VM.Stdlib.Os, + Lua.VM.Stdlib.Debug + ] + @doc """ Installs the standard library into the given VM state. """ @spec install(State.t()) :: State.t() def install(%State{} = state) do - state - |> State.register_function("type", &lua_type/2) - |> State.register_function("tostring", &lua_tostring/2) - |> State.register_function("tonumber", &lua_tonumber/2) - |> State.register_function("print", &lua_print/2) - |> State.register_function("error", &lua_error/2) - |> State.register_function("assert", &lua_assert/2) - |> State.register_function("pcall", &lua_pcall/2) - |> State.register_function("xpcall", &lua_xpcall/2) - |> State.register_function("rawget", &lua_rawget/2) - |> State.register_function("rawset", &lua_rawset/2) - |> State.register_function("rawlen", &lua_rawlen/2) - |> State.register_function("rawequal", &lua_rawequal/2) - |> State.register_function("next", &lua_next/2) - |> State.register_function("pairs", &lua_pairs/2) - |> State.register_function("ipairs", &lua_ipairs/2) - |> State.register_function("setmetatable", &lua_setmetatable/2) - |> State.register_function("getmetatable", &lua_getmetatable/2) - |> State.register_function("select", &lua_select/2) - |> State.register_function("load", &lua_load/2) - |> State.register_function("require", &lua_require/2) - |> State.register_function("collectgarbage", &lua_collectgarbage/2) - |> State.register_function("dofile", &lua_dofile/2) - |> State.set_global("_VERSION", "Lua 5.3") - |> install_package_table() - |> install_library(Lua.VM.Stdlib.String) - |> install_library(Lua.VM.Stdlib.Math) - |> install_library(Lua.VM.Stdlib.Table) - |> install_library(Lua.VM.Stdlib.Utf8) - |> install_library(Lua.VM.Stdlib.Os) - |> install_library(Lua.VM.Stdlib.Debug) - |> preload_stdlib_modules() + {state, loaded} = + state + |> State.set_globals(base_globals()) + |> install_package_table() + + @libraries + |> Enum.reduce(state, &install_library(&2, &1, loaded)) |> install_unpack_alias() |> install_global_g() end - # Install a stdlib library module and register it in package.loaded - defp install_library(state, module) do + # The base globals, in the iteration order `pairs(_G)` reports them. Seeded + # in one `_G` update rather than one per name. + defp base_globals do + [ + {"type", {:native_func, &lua_type/2}}, + {"tostring", {:native_func, &lua_tostring/2}}, + {"tonumber", {:native_func, &lua_tonumber/2}}, + {"print", {:native_func, &lua_print/2}}, + {"error", {:native_func, &lua_error/2}}, + {"assert", {:native_func, &lua_assert/2}}, + {"pcall", {:native_func, &lua_pcall/2}}, + {"xpcall", {:native_func, &lua_xpcall/2}}, + {"rawget", {:native_func, &lua_rawget/2}}, + {"rawset", {:native_func, &lua_rawset/2}}, + {"rawlen", {:native_func, &lua_rawlen/2}}, + {"rawequal", {:native_func, &lua_rawequal/2}}, + {"next", {:native_func, &lua_next/2}}, + {"pairs", {:native_func, &lua_pairs/2}}, + {"ipairs", {:native_func, &lua_ipairs/2}}, + {"setmetatable", {:native_func, &lua_setmetatable/2}}, + {"getmetatable", {:native_func, &lua_getmetatable/2}}, + {"select", {:native_func, &lua_select/2}}, + {"load", {:native_func, &lua_load/2}}, + {"require", {:native_func, &lua_require/2}}, + {"collectgarbage", {:native_func, &lua_collectgarbage/2}}, + {"dofile", {:native_func, &lua_dofile/2}}, + {"_VERSION", "Lua 5.3"} + ] + end + + # Install a stdlib library module and register it in package.loaded. + # + # `loaded` is the `package.loaded` tref, resolved once by the caller — + # rediscovering it per library means re-reading the `package` global and its + # table for every module installed. + defp install_library(state, module, loaded) do state = module.install(state) name = module.lib_name() case State.get_global(state, name) do - {:tref, _} = tref -> cache_module_result(state, name, tref) + {:tref, _} = tref -> cache_in_loaded(state, loaded, name, tref) _ -> state end end @@ -83,34 +103,12 @@ defmodule Lua.VM.Stdlib do defp install_global_g(state) do g_ref = State.g_ref(state) - # Expose _G to Lua under the name "_G" - state = State.set_global(state, "_G", g_ref) - - # _ENV is also exposed at boot for backwards compatibility with code that - # references `_ENV` as a global. The compiler binds `_ENV` as a chunk-level - # local (register 0) at execute time, so this global is mostly for - # introspection; user-level `_ENV` reassignment goes to that local, not + # `_ENV` is exposed alongside `_G` at boot for backwards compatibility with + # code that references `_ENV` as a global. The compiler binds `_ENV` as a + # chunk-level local (register 0) at execute time, so this global is mostly + # for introspection; user-level `_ENV` reassignment goes to that local, not # this global. - State.set_global(state, "_ENV", g_ref) - end - - # Pre-load any stdlib table globals into package.loaded so that - # require("string"), require("math"), etc. resolve to the existing global - # tables without triggering a filesystem search. This mirrors Lua 5.3's - # behaviour where package.loaded is pre-populated by the runtime. - # - # install_library/2 already caches the four installed modules (string, math, - # table, debug), so this pass is a safety net for any future stdlib tables - # that may be added as globals before this call. - defp preload_stdlib_modules(state) do - modules = ["string", "math", "table", "utf8", "debug"] - - Enum.reduce(modules, state, fn name, acc -> - case State.get_global(acc, name) do - {:tref, _} = tref -> cache_module_result(acc, name, tref) - _ -> acc - end - end) + State.set_globals(state, [{"_G", g_ref}, {"_ENV", g_ref}]) end # type(v) — returns the type of v as a string @@ -667,8 +665,10 @@ defmodule Lua.VM.Stdlib do state = State.set_global(state, "package", package_tref) - # Cache "package" itself in package.loaded - cache_module_result(state, "package", package_tref) + # Cache "package" itself in package.loaded, and hand the caller the + # `loaded` tref so the library installs that follow do not each re-resolve + # it through the `package` global. + {cache_in_loaded(state, loaded_tref, "package", package_tref), loaded_tref} end # require(modname) — loads a Lua module @@ -801,9 +801,12 @@ defmodule Lua.VM.Stdlib do # Cache the module result in package.loaded defp cache_module_result(state, modname, result) do - {:tref, loaded_id} = get_package_loaded_ref(state) + cache_in_loaded(state, get_package_loaded_ref(state), modname, result) + end - State.update_table(state, {:tref, loaded_id}, fn loaded_table -> + # Cache the module result in an already-resolved package.loaded table + defp cache_in_loaded(state, loaded_ref, modname, result) do + State.update_table(state, loaded_ref, fn loaded_table -> Table.put(loaded_table, modname, result) end) end diff --git a/lib/lua/vm/table.ex b/lib/lua/vm/table.ex index 5116c5a..2ec39d5 100644 --- a/lib/lua/vm/table.ex +++ b/lib/lua/vm/table.ex @@ -112,6 +112,27 @@ defmodule Lua.VM.Table do end defp split_from_map(table, data) do + case string_keys(:maps.to_list(data), []) do + {:ok, keys} -> hash_only(table, data, keys) + :error -> sorted_fold(table, data) + end + end + + # Every key is a string, so nothing routes to the array, nothing touches the + # border, and no key can be dead in a table this function is allowed to see + # (both callers reset `dead`). That makes the whole build a rename of the + # caller's map plus the iteration order, instead of N struct-rebuilding + # `put/3` calls. + # + # `order_tail` is the descending key list because that is exactly the struct + # the fold produced: it sorted the pairs and `insert_hash` prepended each + # key. Iteration order over a stdlib table is therefore bit-for-bit what it + # has always been. + defp hash_only(table, data, keys) do + %{table | data: data, order: [], order_tail: Enum.sort(keys, :desc), order_index: nil, order_arr: nil} + end + + defp sorted_fold(table, data) do # Sorted fold: integer keys come first, ascending (term order puts numbers # before other keys), so contiguous appends hit put/3's O(1) array path # instead of parking in the hash and draining via absorb_from_hash — an @@ -120,6 +141,15 @@ defmodule Lua.VM.Table do Enum.reduce(Enum.sort(data), table, fn {k, v}, acc -> put(acc, k, v) end) end + # Collects the keys when the map is entirely string-keyed with no `nil` + # values — the shape every stdlib library table has. A `nil` value means + # "absent" in Lua, so a map carrying one has to go through `put/3`'s delete + # branch; anything other than a string key may normalize into the array. + defp string_keys([], keys), do: {:ok, keys} + defp string_keys([{_k, nil} | _rest], _keys), do: :error + defp string_keys([{k, _v} | rest], keys) when is_binary(k), do: string_keys(rest, [k | keys]) + defp string_keys(_pairs, _keys), do: :error + @doc """ Writes `value` into the table under `key`, honoring Lua semantics: diff --git a/mix.exs b/mix.exs index 36bb00c..ff90f21 100644 --- a/mix.exs +++ b/mix.exs @@ -60,6 +60,7 @@ defmodule Lua.MixProject do # autolinking to filtered pages, which errors under # `--warnings-as-errors`. skip_code_autolink_to: [ + "Lua.VM.Bootstrap.reset/0", "Lua.VM.Executor.current_position/0", "Lua.VM.ErrorFormatter.to_map/3", "Lua.VM.RuntimeError", diff --git a/test/lua/vm/bootstrap_test.exs b/test/lua/vm/bootstrap_test.exs new file mode 100644 index 0000000..cd9f5a2 --- /dev/null +++ b/test/lua/vm/bootstrap_test.exs @@ -0,0 +1,286 @@ +defmodule Lua.VM.BootstrapTest do + @moduledoc """ + Pins the contract of the memoized boot templates behind `Lua.new/1`: a VM + built from a shared template must be indistinguishable from one built from + scratch, and must stay isolated from every other VM built from it. + """ + + use ExUnit.Case, async: true + + alias Lua.VM.Bootstrap + alias Lua.VM.Limits + alias Lua.VM.State + + describe "Lua.new/1" do + test "the standard library works" do + lua = Lua.new() + + assert {[2], _} = Lua.eval!(lua, "return 1 + 1") + assert {["HELLO"], _} = Lua.eval!(lua, ~S[return string.upper("hello")]) + assert {[3], _} = Lua.eval!(lua, "return math.max(1, 3, 2)") + assert {["a,b"], _} = Lua.eval!(lua, ~S[return table.concat({"a", "b"}, ",")]) + assert {["Lua 5.3"], _} = Lua.eval!(lua, "return _VERSION") + assert {[true], _} = Lua.eval!(lua, "return _G.print ~= nil") + end + + test "sequential VMs are identical and evaluate identically" do + first = Lua.new() + second = Lua.new() + + assert :erlang.term_to_binary(first) === :erlang.term_to_binary(second) + + script = ~S""" + local acc = {} + for k in pairs(_G) do acc[#acc + 1] = k end + return table.concat(acc, ","), tostring(#acc) + """ + + assert {results, _} = Lua.eval!(first, script) + assert {^results, _} = Lua.eval!(second, script) + end + + test "a fresh VM starts with the boot table and global counts" do + state = Lua.new().state + + assert state.g_ref == {:tref, 0} + assert state.table_next_id == 12 + assert map_size(state.tables) == 12 + end + + test "mutating one VM does not leak into the next" do + mutated = + Lua.new() + |> Lua.set!([:planted], "leaked") + |> Lua.set!([:deeply, :nested], "leaked") + + assert {["leaked"], _} = Lua.eval!(mutated, "return planted") + + fresh = Lua.new() + + assert {[nil], _} = Lua.eval!(fresh, "return planted") + assert {[nil], _} = Lua.eval!(fresh, "return deeply") + assert fresh.state.table_next_id == 12 + end + + test "globals and stdlib tables written from Lua do not leak into the next VM" do + {_, mutated} = + Lua.eval!(Lua.new(), ~S""" + planted = "leaked" + string.shout = function(s) return s end + """) + + assert {["leaked", true], _} = Lua.eval!(mutated, "return planted, string.shout ~= nil") + + assert {[nil, nil], _} = Lua.eval!(Lua.new(), "return planted, string.shout") + end + + test "limits and :debug are per-VM, not baked into the template" do + limited = Lua.new(max_call_depth: 10, max_string_bytes: 1024, max_instructions: 5000, debug: true) + + assert limited.debug + assert limited.state.max_call_depth == 10 + assert limited.state.max_string_bytes == 1024 + assert limited.state.max_instructions == 5000 + + default = Lua.new() + + refute default.debug + assert default.state.max_call_depth == :infinity + assert default.state.max_string_bytes == Limits.max_string_bytes() + assert default.state.max_instructions == :infinity + end + end + + describe "sandbox options" do + test "the default deny-list still applies" do + lua = Lua.new() + + assert_raise Lua.RuntimeException, "Lua runtime error: os.exit(_) is sandboxed", fn -> + Lua.eval!(lua, "os.exit(1)") + end + + assert_raise Lua.RuntimeException, "Lua runtime error: require(_) is sandboxed", fn -> + Lua.eval!(lua, ~S[require("anything")]) + end + end + + test "sandboxed: [] leaves the library intact" do + lua = Lua.new(sandboxed: []) + + assert {["required file successfully"], _} = + Lua.eval!(lua, ~S""" + package.path = "./test/fixtures/?.lua" + return require("test_require") + """) + end + + test "a custom deny-list sandboxes only what it names" do + lua = Lua.new(sandboxed: [[:os, :exit]]) + + assert_raise Lua.RuntimeException, "Lua runtime error: os.exit(_) is sandboxed", fn -> + Lua.eval!(lua, "os.exit(1)") + end + + assert {[true], _} = Lua.eval!(lua, "return load ~= nil") + assert {[true], _} = Lua.eval!(lua, "return package ~= nil") + end + + test ":exclude lifts entries out of the default deny-list" do + lua = Lua.new(exclude: [[:require], [:package]]) + + assert {["required file successfully"], _} = + Lua.eval!(Lua.set_lua_paths(lua, "./test/fixtures/?.lua"), ~S[return require("test_require")]) + + assert_raise Lua.RuntimeException, "Lua runtime error: os.exit(_) is sandboxed", fn -> + Lua.eval!(lua, "os.exit(1)") + end + end + + test "custom-sandbox VMs do not disturb the default VM" do + _custom = Lua.set!(Lua.new(sandboxed: []), [:planted], "leaked") + + assert {[nil], _} = Lua.eval!(Lua.new(), "return planted") + + assert_raise Lua.RuntimeException, "Lua runtime error: os.exit(_) is sandboxed", fn -> + Lua.eval!(Lua.new(), "os.exit(1)") + end + end + end + + describe "fetch/2" do + setup do + key = {__MODULE__, System.unique_integer()} + on_exit(fn -> :persistent_term.erase(key) end) + %{key: key} + end + + test "builds once and returns the same term afterwards", %{key: key} do + test = self() + + builder = fn -> + send(test, :built) + %{value: State.new()} + end + + first = Bootstrap.fetch(key, builder) + second = Bootstrap.fetch(key, builder) + + assert first === second + assert_received :built + refute_received :built + end + + test "keys are independent", %{key: key} do + other = {__MODULE__, System.unique_integer()} + on_exit(fn -> :persistent_term.erase(other) end) + + assert Bootstrap.fetch(key, fn -> :one end) == :one + assert Bootstrap.fetch(other, fn -> :two end) == :two + assert Bootstrap.fetch(key, fn -> :three end) == :one + end + + test "a stale fingerprint rebuilds the term", %{key: key} do + :persistent_term.put(key, {[{__MODULE__, <<"not the current md5">>}], :stale}) + + assert Bootstrap.fetch(key, fn -> :rebuilt end) == :rebuilt + assert Bootstrap.fetch(key, fn -> :again end) == :rebuilt + end + + test "a fingerprinted module that no longer exists rebuilds instead of raising", %{key: key} do + :persistent_term.put(key, {[{Lua.VM.NoSuchModule, <<"md5 of deleted code">>}], :stale}) + + assert Bootstrap.fetch(key, fn -> :rebuilt end) == :rebuilt + assert Bootstrap.fetch(key, fn -> :again end) == :rebuilt + end + + test "the fingerprint covers template-shaping modules even without captured funs", %{key: key} do + # The stored term carries no closures at all, so every module below gets + # into the fingerprint only via the explicit template-module list: a + # struct-layout or default-limit change in any of them must invalidate + # the template even though no fun in it points there. + assert :code.get_mode() == :interactive + + Bootstrap.fetch(key, fn -> %{value: :no_funs_here} end) + + {fingerprint, _term} = :persistent_term.get(key) + modules = Enum.map(fingerprint, fn {module, _md5} -> module end) + + for module <- [Lua, Bootstrap, Limits, State, Lua.VM.Stdlib, Lua.VM.Table] do + assert module in modules, "fingerprint is missing template-shaping module #{inspect(module)}" + end + end + end + + describe "reset/0" do + test "erases every memoized key so the next fetch rebuilds" do + _warm = Lua.new() + + for key <- Bootstrap.memoized_keys() do + assert :persistent_term.get(key, :missing) != :missing, + "expected #{inspect(key)} to be memoized by Lua.new/1" + end + + assert Bootstrap.reset() == :ok + + for key <- Bootstrap.memoized_keys() do + assert :persistent_term.get(key, :missing) == :missing + end + + assert {[2], _} = Lua.eval!(Lua.new(), "return 1 + 1") + end + + test "erases every memoized key populated by racing cold starts" do + Bootstrap.reset() + + test = self() + + workers = + for _ <- 1..64 do + Task.async(fn -> + send(test, {:ready, self()}) + assert_receive :go + Lua.new() + end) + end + + for %Task{pid: pid} <- workers, do: assert_receive({:ready, ^pid}) + for %Task{pid: pid} <- workers, do: send(pid, :go) + + Task.await_many(workers) + + assert Bootstrap.reset() == :ok + + for key <- Bootstrap.memoized_keys() do + assert :persistent_term.get(key, :missing) == :missing, + "reset/0 left #{inspect(key)} behind after a concurrent cold start" + end + end + + test "does not consult runtime bookkeeping, and erases the obsolete entry" do + # An entry left by a build that tracked the stored keys at runtime. Such + # tracking is a read-modify-write that concurrent cold starts can + # truncate, so a truncated list must not be able to narrow what `reset/0` + # erases — and the entry itself must not survive the reset. + obsolete = {Bootstrap, :keys} + on_exit(fn -> :persistent_term.erase(obsolete) end) + + _warm = Lua.new() + :persistent_term.put(obsolete, []) + + assert Bootstrap.reset() == :ok + + for key <- [obsolete | Bootstrap.memoized_keys()] do + assert :persistent_term.get(key, :missing) == :missing, + "reset/0 left #{inspect(key)} behind" + end + end + + test "Lua.new/1 still works after a reset" do + _warm = Lua.new() + + assert Bootstrap.reset() == :ok + + assert {[2], _} = Lua.eval!(Lua.new(), "return 1 + 1") + end + end +end diff --git a/test/lua/vm/table_iteration_test.exs b/test/lua/vm/table_iteration_test.exs index 73ad9d2..86a3c6d 100644 --- a/test/lua/vm/table_iteration_test.exs +++ b/test/lua/vm/table_iteration_test.exs @@ -42,6 +42,15 @@ defmodule Lua.VM.TableIterationTest do defp entries_gen, do: list_of(tuple({key_gen(), integer(1..1000)}), max_length: 40) + # Values for the map-based constructors (`from_data/1`, `replace_data/2`): + # unlike `entries_gen/0` these include `nil`, which means "absent" in Lua and + # forces the constructors off the all-string one-shot build (its nil bail-out + # guard) onto the put/3 fold with its delete branch. + defp data_entries_gen do + value_gen = frequency([{4, integer(1..1000)}, {1, constant(nil)}]) + list_of(tuple({key_gen(), value_gen}), max_length: 40) + end + # Builds the table and an independent key=>value oracle from the same ops # (last write wins on both sides; generated values are always non-nil). defp build_pair(ops) do @@ -320,6 +329,55 @@ defmodule Lua.VM.TableIterationTest do end end + property "from_data equals the sorted put/3 fold it replaces, key shape regardless" do + # from_data/1 skips the fold entirely for an all-string-keyed map, + # building `data` and `order` in one shot. That shortcut has to be + # invisible: the resulting struct must be the one the fold produced, + # field for field, so iteration order and border bookkeeping are + # unchanged. Generated values include nil, so maps carrying a + # nil-valued entry must bail out of the one-shot build and take the + # fold's delete branch, matching it exactly. + check all(entries <- data_entries_gen()) do + data = Map.new(entries) + folded = Enum.reduce(Enum.sort(data), %Table{}, fn {k, v}, acc -> Table.put(acc, k, v) end) + + assert Table.from_data(data) == folded + end + end + + property "replace_data on a lived-in table equals replace_data on a fresh one" do + # replace_data/2 is the riskier split_from_map/2 caller: the struct it + # rebuilds arrives pre-populated with an array/hash split, dead keys, + # an order memo, and a metatable. Nothing from that history may leak + # into the rebuilt contents — only the metatable survives — so the + # result must be indistinguishable from replacing into a table that + # never held anything. + check all(ops <- entries_gen(), entries <- data_entries_gen()) do + {built, oracle} = build_pair(ops) + + lived_in = + oracle + |> Map.keys() + |> Enum.take(div(map_size(oracle), 2)) + |> Enum.reduce(built, fn k, acc -> Table.put(acc, k, nil) end) + |> Table.flush_order() + |> Map.put(:metatable, {:tref, 7}) + + data = Map.new(entries) + replaced = Table.replace_data(lived_in, data) + + assert replaced == Table.replace_data(%Table{metatable: {:tref, 7}}, data) + assert replaced.metatable == {:tref, 7} + assert replaced.dead == %{} + + # The walk sees exactly the non-nil entries of the new data map. + live = for {k, v} <- data, v != nil, into: %{}, do: {k, v} + walked = walk(Table.flush_order(replaced)) + + assert MapSet.new(walked) == MapSet.new(live) + end + end + property "clearing the current key mid-walk still visits every live key once (§6.1), memo and fallback" do check all(ops <- entries_gen()) do {table, oracle} = build_pair(ops)