diff --git a/lib/lua/vm/stdlib.ex b/lib/lua/vm/stdlib.ex index b3a8e44a..9c6a54c9 100644 --- a/lib/lua/vm/stdlib.ex +++ b/lib/lua/vm/stdlib.ex @@ -339,18 +339,16 @@ defmodule Lua.VM.Stdlib do key = List.first(rest) table = Map.fetch!(state.tables, id) - # Eagerly flush any deferred-append `order_tail` so subsequent - # iteration steps don't re-merge on every call. The first call to - # `lua_next` for a given iteration pays the cost once; the rest see - # a clean `order` list. + # Eagerly flush any deferred-append `order_tail` and build the O(1) + # iteration memo so subsequent steps are index lookups rather than + # linear scans. The first call to `lua_next` for a given iteration + # pays the cost once; the rest see a clean `order` and a live memo. {table, state} = - case table.order_tail do - [] -> - {table, state} - - _ -> - flushed = Table.flush_order(table) - {flushed, %{state | tables: Map.put(state.tables, id, flushed)}} + if table.order_tail == [] and table.order_index != nil do + {table, state} + else + flushed = Table.flush_order(table) + {flushed, %{state | tables: Map.put(state.tables, id, flushed)}} end case Table.next_entry(table, key) do diff --git a/lib/lua/vm/table.ex b/lib/lua/vm/table.ex index 5a9dacf5..9f91c465 100644 --- a/lib/lua/vm/table.ex +++ b/lib/lua/vm/table.ex @@ -33,6 +33,8 @@ defmodule Lua.VM.Table do data: %{}, order: [], order_tail: [], + order_index: nil, + order_arr: nil, dead: %{}, metatable: nil @@ -42,6 +44,8 @@ defmodule Lua.VM.Table do data: %{optional(term()) => term()}, order: list(term()), order_tail: list(term()), + order_index: %{optional(term()) => non_neg_integer()} | nil, + order_arr: :array.array() | nil, dead: %{optional(term()) => true}, metatable: {:tref, non_neg_integer()} | nil } @@ -72,7 +76,20 @@ defmodule Lua.VM.Table do """ @spec replace_data(t(), map()) :: t() def replace_data(%__MODULE__{} = table, data) when is_map(data) do - split_from_map(%{table | arr: :undefined, arr_n: 0, data: %{}, order: [], order_tail: [], dead: %{}}, data) + split_from_map( + %{ + table + | arr: :undefined, + arr_n: 0, + data: %{}, + order: [], + order_tail: [], + order_index: nil, + order_arr: nil, + dead: %{} + }, + data + ) end defp split_from_map(table, data) do @@ -141,6 +158,8 @@ defmodule Lua.VM.Table do | data: Map.put(data, key, value), order: new_order, order_tail: [key], + order_index: nil, + order_arr: nil, dead: Map.delete(dead, key) } @@ -148,7 +167,13 @@ defmodule Lua.VM.Table do %{table | data: Map.put(data, key, value)} true -> - %{table | data: Map.put(data, key, value), order_tail: [key | order_tail]} + %{ + table + | data: Map.put(data, key, value), + order_tail: [key | order_tail], + order_index: nil, + order_arr: nil + } end end @@ -182,7 +207,9 @@ defmodule Lua.VM.Table do table | data: Map.delete(data, key), order: Enum.reject(order, &(&1 === key)), - order_tail: Enum.reject(tail, &(&1 === key)) + order_tail: Enum.reject(tail, &(&1 === key)), + order_index: nil, + order_arr: nil } end @@ -371,7 +398,7 @@ defmodule Lua.VM.Table do @spec next_entry(t(), term()) :: {term(), term()} | nil | :invalid_key def next_entry(%__MODULE__{} = table, nil) do case first_array_live(table, 1) do - nil -> first_live(merged_order(table), table.data) + nil -> first_hash_live(table) pair -> pair end end @@ -381,13 +408,52 @@ defmodule Lua.VM.Table do if is_integer(key) and key >= 1 and key <= n do case first_array_live(table, key + 1) do - nil -> first_live(merged_order(table), table.data) + nil -> first_hash_live(table) pair -> pair end else - case advance_past(merged_order(table), key) do - :not_found -> :invalid_key - remaining -> first_live(remaining, table.data) + next_hash_after(table, key) + end + end + + # First live hash entry at or after the start of iteration order. Uses the + # memoized order array when present (built once per iteration in + # `flush_order/1`); otherwise falls back to the structural `order` list. + defp first_hash_live(%__MODULE__{order_arr: nil} = table) do + first_live(merged_order(table), table.data) + end + + defp first_hash_live(%__MODULE__{order_arr: arr, data: data}) do + first_live_from(arr, 0, data) + end + + # First live hash entry strictly after `key`. With the memo present this is + # an O(1) index lookup plus a forward scan to the next live key; the scan + # only ever advances, so a full `pairs` loop is O(n) total. Without the memo + # it falls back to the linear `advance_past/2` scan. + defp next_hash_after(%__MODULE__{order_index: nil} = table, key) do + case advance_past(merged_order(table), key) do + :not_found -> :invalid_key + remaining -> first_live(remaining, table.data) + end + end + + defp next_hash_after(%__MODULE__{order_index: index, order_arr: arr, data: data}, key) do + case Map.get(index, key) do + nil -> :invalid_key + idx -> first_live_from(arr, idx + 1, data) + end + end + + defp first_live_from(arr, idx, data) do + if idx >= :array.size(arr) do + nil + else + k = :array.get(idx, arr) + + case Map.fetch(data, k) do + {:ok, v} -> {k, v} + :error -> first_live_from(arr, idx + 1, data) end end end @@ -402,13 +468,23 @@ defmodule Lua.VM.Table do end @doc """ - Flushes any pending `order_tail` appends into `order`. Idempotent. + Flushes any pending `order_tail` appends into `order` and (re)builds the + O(1)-lookup iteration memo (`order_arr`/`order_index`). Idempotent. + + Called once at the start of a `pairs`/`next` iteration so the per-step + hash advance is an index lookup rather than a linear scan. The memo is + invalidated (set to `nil`) by any structural mutation of `order`, so a + flush with an already-empty tail still rebuilds it when missing. """ @spec flush_order(t()) :: t() - def flush_order(%__MODULE__{order_tail: []} = table), do: table + def flush_order(%__MODULE__{order_tail: [], order_index: index} = table) when index != nil do + table + end def flush_order(%__MODULE__{order: order, order_tail: tail} = table) do - %{table | order: order ++ Enum.reverse(tail), order_tail: []} + flushed = order ++ Enum.reverse(tail) + index = flushed |> Enum.with_index() |> Map.new() + %{table | order: flushed, order_tail: [], order_arr: :array.from_list(flushed), order_index: index} end defp merged_order(%__MODULE__{order: order, order_tail: []}), do: order diff --git a/test/lua/vm/table_iteration_test.exs b/test/lua/vm/table_iteration_test.exs new file mode 100644 index 00000000..47cbd3ed --- /dev/null +++ b/test/lua/vm/table_iteration_test.exs @@ -0,0 +1,247 @@ +defmodule Lua.VM.TableIterationTest do + @moduledoc """ + Pins the O(1)-per-step hash iteration path and its equivalence with the + list-based fallback, while preserving Lua 5.3 §6.1 dead-key semantics. + + `Lua.VM.Table.flush_order/1` builds a memoized order array plus a + `key => index` map so each `next_entry/2` hash step is an index lookup + and a forward scan to the next live key, rather than a linear `===` scan + over the whole order list. Both the memoized path (after `flush_order/1`) + and the unflushed fallback must agree on `{k, v}` / `nil` / `:invalid_key`. + """ + + use ExUnit.Case, async: true + use ExUnitProperties + + alias Lua.VM.Table + + # Collects the full hash-key walk starting from `nil`, driving the same + # `Table` instance `next_entry/2` would see during a real `pairs` loop. + defp walk(table) do + nil + |> Stream.unfold(fn key -> + case Table.next_entry(table, key) do + nil -> nil + :invalid_key -> nil + {k, _v} = pair -> {pair, k} + end + end) + |> Enum.to_list() + end + + # Mix of array keys, sparse-integer hash keys, and string hash keys, so the + # array arm, the integer-hash arm, and the string-hash arm of next_entry/2 + # all get exercised in one table. + defp key_gen do + frequency([ + {5, integer(1..12)}, + {2, integer(100..110)}, + {5, string(?a..?z, min_length: 1, max_length: 3)} + ]) + end + + defp entries_gen, do: list_of(tuple({key_gen(), integer(1..1000)}), max_length: 40) + + # Builds the table and an independent key=>value oracle from the same ops + # (last write wins on both sides; generated values are always non-nil). + defp build_pair(ops) do + Enum.reduce(ops, {%Table{}, %{}}, fn {k, v}, {t, m} -> + {Table.put(t, k, v), Map.put(m, k, v)} + end) + end + + # Walks like `walk/1`, but deletes the just-returned key before resuming + # from it — the canonical Lua "remove the current field during traversal" + # pattern. §6.1 guarantees the cleared key stays a valid resume point, so + # this must visit every originally-live key once and never see :invalid_key. + defp walk_deleting(table), do: walk_deleting(table, nil, []) + + defp walk_deleting(table, key, acc) do + case Table.next_entry(table, key) do + nil -> + Enum.reverse(acc) + + :invalid_key -> + flunk("next_entry returned :invalid_key resuming from just-cleared key #{inspect(key)}") + + {k, _v} -> + walk_deleting(Table.put(table, k, nil), k, [k | acc]) + end + end + + describe "memoized iteration path" do + test "flush_order builds order_arr and order_index covering every live key" do + table = + 1..50 + |> Enum.reduce(%Table{}, fn i, acc -> + Table.put(acc, "k#{i}", i) + end) + |> Table.flush_order() + + assert table.order_index + assert table.order_arr + assert :array.size(table.order_arr) == 50 + assert map_size(table.order_index) == 50 + end + + test "a full walk over a large string-keyed table visits every key once" do + keys = Enum.map(1..200, &"k#{&1}") + + table = + keys + |> Enum.reduce(%Table{}, fn k, acc -> Table.put(acc, k, k) end) + |> Table.flush_order() + + walked = walk(table) + + assert Enum.map(walked, &elem(&1, 0)) == keys + assert length(walked) == 200 + end + + test "next_entry on a key absent everywhere returns :invalid_key via the memo" do + table = + %Table{} + |> Table.put("a", 1) + |> Table.put("b", 2) + |> Table.flush_order() + + assert Table.next_entry(table, "ghost") == :invalid_key + end + + test "a cleared key stays a valid resume point (§6.1)" do + table = + %Table{} + |> Table.put("a", 1) + |> Table.put("b", 2) + |> Table.put("c", 3) + |> Table.flush_order() + + # Clear "b" mid-iteration; it must remain in order_arr so next(t, "b") + # advances to the following live key rather than raising. + cleared = Table.put(table, "b", nil) + + assert Table.next_entry(cleared, "b") == {"c", 3} + end + end + + describe "memoized path equals list-based fallback" do + test "for every key, the memoized result matches the unflushed result" do + base = + %Table{} + |> Table.put(1, "one") + |> Table.put("x", "ex") + |> Table.put(2, "two") + |> Table.put("y", "why") + |> Table.put(7, "seven") + |> Table.put("z", "zee") + + flushed = Table.flush_order(base) + + # The unflushed `base` still has a pending order_tail and a nil memo, + # so its next_entry takes the merged_order/advance_past fallback. + assert base.order_index == nil + + keys_to_probe = [nil, 1, 2, 7, "x", "y", "z", "ghost"] + + for key <- keys_to_probe do + assert Table.next_entry(flushed, key) == Table.next_entry(base, key), + "memo and fallback disagree on next_entry for key #{inspect(key)}" + end + end + end + + describe "memo invalidation across mutations" do + test "inserting a new key clears the memo so a fresh walk includes it" do + table = + %Table{} + |> Table.put("a", 1) + |> Table.put("b", 2) + |> Table.flush_order() + + assert table.order_index + + mutated = Table.put(table, "c", 3) + assert mutated.order_index == nil + assert mutated.order_arr == nil + + reflushed = Table.flush_order(mutated) + assert Enum.map(walk(reflushed), &elem(&1, 0)) == ["a", "b", "c"] + end + + test "dead-key revival rebuilds the memo with the revived position" do + table = + %Table{} + |> Table.put("a", 1) + |> Table.put("b", 2) + |> Table.flush_order() + + revived = + table + |> Table.put("a", nil) + |> Table.put("a", 100) + + assert revived.order_index == nil + + reflushed = Table.flush_order(revived) + walked = walk(reflushed) + + assert {"a", 100} in walked + assert {"b", 2} in walked + assert length(walked) == 2 + end + + test "clearing a value does not invalidate the memo (§6.1 resume point)" do + table = + %Table{} + |> Table.put("a", 1) + |> Table.put("b", 2) + |> Table.flush_order() + + # Plain delete of a hash key only marks it dead; it must NOT touch the + # order/memo, so the cleared key remains reachable for `next`. + cleared = Table.put(table, "a", nil) + + assert cleared.order_index + assert cleared.order_arr == table.order_arr + end + end + + describe "iteration properties (StreamData)" do + property "a full walk visits exactly the live key set, once each, with matching values" do + check all(ops <- entries_gen()) do + {table, oracle} = build_pair(ops) + walked = walk(Table.flush_order(table)) + keys = Enum.map(walked, &elem(&1, 0)) + + # No duplicates and nothing missing. + assert length(keys) == map_size(oracle) + assert MapSet.new(keys) == MapSet.new(Map.keys(oracle)) + + for {k, v} <- walked do + assert v == Map.fetch!(oracle, k), "value for #{inspect(k)} diverged from the oracle" + end + end + end + + property "the memoized walk equals the unflushed list-based fallback walk" do + check all(ops <- entries_gen()) do + {table, _oracle} = build_pair(ops) + assert walk(Table.flush_order(table)) == walk(table) + end + end + + property "clearing the current key mid-walk still visits every live key once (§6.1), memo and fallback" do + check all(ops <- entries_gen()) do + {table, oracle} = build_pair(ops) + expected = MapSet.new(Map.keys(oracle)) + + for start <- [table, Table.flush_order(table)] do + visited = walk_deleting(start) + + assert length(visited) == map_size(oracle) + assert MapSet.new(visited) == expected + end + end + end + end +end