From ad22656f687238f53633c50ae4f6ed69d8c3df6a Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Wed, 27 May 2026 22:36:11 -0400 Subject: [PATCH 1/2] feat(stdlib): add utf8 library and align integer-arith error wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the Lua 5.3 §6.5 `utf8` standard library — `char`, `codepoint`, `codes`, `len`, `offset`, `charpattern` — covering the 0x0..0x10FFFF range with overlong/invalid-sequence detection. The official suite's `utf8.lua` now passes outright and is promoted out of the skip list. Also normalises integer-arithmetic error phrasing to match PUC-Lua: "attempt to perform 'n//0'" (was "attempt to divide by zero") and "attempt to perform 'n%0'" (was "attempt to perform modulo by zero"). Adds an `ArgumentError.wrong_number_of_arguments/1` helper as the shared constructor for the variadic-stdlib runtime-error template. Each touched template is pinned with a unit test in `test/lua/vm/error_format_test.exs`. Closes #253. Progresses #252 (the listed `n//0`, `n%0`, and "wrong number of arguments" templates; field-hint threading on arithmetic sites deferred to a follow-up — requires plumbing target hints through all `raise_arith_type_error` callsites). --- lib/lua/vm/argument_error.ex | 15 + lib/lua/vm/executor.ex | 4 +- lib/lua/vm/stdlib.ex | 3 +- lib/lua/vm/stdlib/table.ex | 2 +- lib/lua/vm/stdlib/utf8.ex | 435 ++++++++++++++++++++++++++++ test/lua/vm/arithmetic_test.exs | 6 +- test/lua/vm/error_format_test.exs | 90 ++++++ test/lua/vm/float_div_zero_test.exs | 8 +- test/lua/vm/stdlib/utf8_test.exs | 222 ++++++++++++++ test/lua53_skips.exs | 8 - 10 files changed, 774 insertions(+), 19 deletions(-) create mode 100644 lib/lua/vm/stdlib/utf8.ex create mode 100644 test/lua/vm/error_format_test.exs create mode 100644 test/lua/vm/stdlib/utf8_test.exs diff --git a/lib/lua/vm/argument_error.ex b/lib/lua/vm/argument_error.ex index 94eba815..979edeb2 100644 --- a/lib/lua/vm/argument_error.ex +++ b/lib/lua/vm/argument_error.ex @@ -139,4 +139,19 @@ defmodule Lua.VM.ArgumentError do got: got ) end + + @doc """ + Builds the PUC-Lua "wrong number of arguments to 'X'" runtime error. + + This is not a bad-argument error — it is the top-level message PUC-Lua's + `luaL_error(L, "wrong number of arguments to '%s'", name)` emits when a + variadic stdlib function is called with too few or too many positional + arguments. Returns a `Lua.VM.RuntimeError` so callers can `raise` it + directly: + + raise ArgumentError.wrong_number_of_arguments("insert") + """ + def wrong_number_of_arguments(function_name) do + Lua.VM.RuntimeError.exception(value: "wrong number of arguments to '#{function_name}'") + end end diff --git a/lib/lua/vm/executor.ex b/lib/lua/vm/executor.ex index 17173297..07491795 100644 --- a/lib/lua/vm/executor.ex +++ b/lib/lua/vm/executor.ex @@ -2321,7 +2321,7 @@ defmodule Lua.VM.Executor do {:ok, nb} <- to_number(b) do cond do is_integer(nb) and nb == 0 -> - raise RuntimeError, value: "attempt to divide by zero", line: line, source: source + raise RuntimeError, value: "attempt to perform 'n//0'", line: line, source: source is_integer(na) and is_integer(nb) -> Numeric.to_signed_int64(lua_idiv(na, nb)) @@ -2349,7 +2349,7 @@ defmodule Lua.VM.Executor do {:ok, nb} <- to_number(b) do cond do is_integer(nb) and nb == 0 -> - raise RuntimeError, value: "attempt to perform modulo by zero", line: line, source: source + raise RuntimeError, value: "attempt to perform 'n%0'", line: line, source: source is_integer(na) and is_integer(nb) -> Numeric.to_signed_int64(na - lua_idiv(na, nb) * nb) diff --git a/lib/lua/vm/stdlib.ex b/lib/lua/vm/stdlib.ex index 5f11c46d..c2b054a3 100644 --- a/lib/lua/vm/stdlib.ex +++ b/lib/lua/vm/stdlib.ex @@ -48,6 +48,7 @@ defmodule Lua.VM.Stdlib do |> install_library(Lua.VM.Stdlib.String) |> install_library(Lua.VM.Stdlib.Math) |> install_library(Lua.VM.Stdlib.Table) + |> install_library(Lua.VM.Stdlib.Utf8) |> install_library(Lua.VM.Stdlib.Debug) |> preload_stdlib_modules() |> install_unpack_alias() @@ -98,7 +99,7 @@ defmodule Lua.VM.Stdlib do # table, debug), so this pass is a safety net for any future stdlib tables # that may be added as globals before this call. defp preload_stdlib_modules(state) do - modules = ["string", "math", "table", "debug"] + modules = ["string", "math", "table", "utf8", "debug"] Enum.reduce(modules, state, fn name, acc -> case State.get_global(acc, name) do diff --git a/lib/lua/vm/stdlib/table.ex b/lib/lua/vm/stdlib/table.ex index 916842ff..ba8fc3b9 100644 --- a/lib/lua/vm/stdlib/table.ex +++ b/lib/lua/vm/stdlib/table.ex @@ -84,7 +84,7 @@ defmodule Lua.VM.Stdlib.Table do end defp table_insert(args, _state) when length(args) > 3 do - raise Lua.VM.RuntimeError, value: "wrong number of arguments to 'insert'" + raise ArgumentError.wrong_number_of_arguments("insert") end defp table_insert([tref | _], _state) do diff --git a/lib/lua/vm/stdlib/utf8.ex b/lib/lua/vm/stdlib/utf8.ex new file mode 100644 index 00000000..405137ea --- /dev/null +++ b/lib/lua/vm/stdlib/utf8.ex @@ -0,0 +1,435 @@ +defmodule Lua.VM.Stdlib.Utf8 do + @moduledoc """ + Lua 5.3 `utf8` standard library (§6.5). + + Operates over byte strings; Lua strings have no Unicode awareness of their + own — this library treats the bytes as a UTF-8 encoded sequence and + validates per the BMP+supplementary range `[0, 0x10FFFF]`. Overlong + encodings (e.g. `\\xC0\\x80` for U+0000), continuation bytes appearing + in the lead position, and codepoints above `0x10FFFF` all surface as + `"invalid UTF-8 code"`. + + ## Functions + + - `utf8.char(...)` — codepoints to UTF-8 string + - `utf8.codepoint(s [, i [, j]])` — UTF-8 string slice to codepoints + - `utf8.codes(s)` — stateless `(byte_pos, codepoint)` iterator + - `utf8.len(s [, i [, j]])` — codepoint count, or `nil, byte_pos` on + the first invalid sequence in the slice + - `utf8.offset(s, n [, i])` — byte position of the n-th codepoint + - `utf8.charpattern` — Lua pattern matching one UTF-8 byte sequence + """ + + @behaviour Lua.VM.Stdlib.Library + + import Bitwise + + alias Lua.VM.ArgumentError + alias Lua.VM.RuntimeError + alias Lua.VM.State + alias Lua.VM.Stdlib.Util + + @charpattern "[\0-\x7F\xC2-\xFD][\x80-\xBF]*" + @max_codepoint 0x10FFFF + + @impl true + def lib_name, do: "utf8" + + @impl true + def install(state) do + utf8_table = %{ + "char" => {:native_func, &utf8_char/2}, + "codepoint" => {:native_func, &utf8_codepoint/2}, + "codes" => {:native_func, &utf8_codes/2}, + "len" => {:native_func, &utf8_len/2}, + "offset" => {:native_func, &utf8_offset/2}, + "charpattern" => @charpattern + } + + {tref, state} = State.alloc_table(state, utf8_table) + State.set_global(state, "utf8", tref) + end + + # --------------------------------------------------------------------------- + # utf8.char(...) + # --------------------------------------------------------------------------- + + defp utf8_char(args, state) do + iolist = + args + |> Enum.with_index(1) + |> Enum.map(fn {cp, idx} -> encode_one(cp, idx) end) + + {[IO.iodata_to_binary(iolist)], state} + end + + defp encode_one(cp, _idx) when is_integer(cp) and cp >= 0 and cp <= @max_codepoint do + encode_codepoint(cp) + end + + defp encode_one(cp, idx) when is_integer(cp) do + raise ArgumentError, + function_name: "utf8.char", + arg_num: idx, + details: "value out of range" + end + + defp encode_one(v, idx) do + raise ArgumentError, + function_name: "utf8.char", + arg_num: idx, + expected: "number", + got: Util.typeof(v) + end + + defp encode_codepoint(cp) when cp < 0x80, do: <> + + defp encode_codepoint(cp) when cp < 0x800 do + <<0xC0 ||| cp >>> 6, 0x80 ||| (cp &&& 0x3F)>> + end + + defp encode_codepoint(cp) when cp < 0x10000 do + <<0xE0 ||| cp >>> 12, 0x80 ||| (cp >>> 6 &&& 0x3F), 0x80 ||| (cp &&& 0x3F)>> + end + + defp encode_codepoint(cp) do + <<0xF0 ||| cp >>> 18, 0x80 ||| (cp >>> 12 &&& 0x3F), 0x80 ||| (cp >>> 6 &&& 0x3F), 0x80 ||| (cp &&& 0x3F)>> + end + + # --------------------------------------------------------------------------- + # utf8.codepoint(s [, i [, j]]) + # --------------------------------------------------------------------------- + + defp utf8_codepoint([], _state) do + raise ArgumentError.value_expected("utf8.codepoint", 1) + end + + defp utf8_codepoint([s | rest], state) do + if !is_binary(s) do + raise ArgumentError, + function_name: "utf8.codepoint", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + + len = byte_size(s) + i = check_integer(Enum.at(rest, 0, 1), "utf8.codepoint", 2) + j = check_integer(Enum.at(rest, 1, i), "utf8.codepoint", 3) + + posi = u_posrelat(i, len) + posj = u_posrelat(j, len) + + if posi < 1 do + raise ArgumentError, + function_name: "utf8.codepoint", + arg_num: 2, + details: "out of range" + end + + if posj > len do + raise ArgumentError, + function_name: "utf8.codepoint", + arg_num: 3, + details: "out of range" + end + + {collect_codepoints(s, posi, posj, []), state} + end + + defp collect_codepoints(_s, posi, posj, acc) when posi > posj do + Enum.reverse(acc) + end + + defp collect_codepoints(s, posi, posj, acc) do + case decode_at(s, posi) do + {cp, len} -> collect_codepoints(s, posi + len, posj, [cp | acc]) + :invalid -> raise RuntimeError, value: "invalid UTF-8 code" + end + end + + # --------------------------------------------------------------------------- + # utf8.len(s [, i [, j]]) + # --------------------------------------------------------------------------- + + defp utf8_len([], _state) do + raise ArgumentError.value_expected("utf8.len", 1) + end + + defp utf8_len([s | rest], state) do + if !is_binary(s) do + raise ArgumentError, + function_name: "utf8.len", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + + len = byte_size(s) + i = check_integer(Enum.at(rest, 0, 1), "utf8.len", 2) + j = check_integer(Enum.at(rest, 1, -1), "utf8.len", 3) + + posi = u_posrelat(i, len) + posj = u_posrelat(j, len) + + if posi < 1 or posi > len + 1 do + raise ArgumentError, + function_name: "utf8.len", + arg_num: 2, + details: "initial position out of string" + end + + if posj > len do + raise ArgumentError, + function_name: "utf8.len", + arg_num: 3, + details: "final position out of string" + end + + {do_utf8_len(s, posi, posj, 0), state} + end + + defp do_utf8_len(_s, posi, posj, n) when posi > posj, do: [n] + + defp do_utf8_len(s, posi, posj, n) do + case decode_at(s, posi) do + {_cp, len} -> do_utf8_len(s, posi + len, posj, n + 1) + :invalid -> [nil, posi] + end + end + + # --------------------------------------------------------------------------- + # utf8.offset(s, n [, i]) + # --------------------------------------------------------------------------- + + defp utf8_offset([], _state) do + raise ArgumentError.value_expected("utf8.offset", 1) + end + + defp utf8_offset([s], _state) do + if !is_binary(s) do + raise ArgumentError, + function_name: "utf8.offset", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + + raise ArgumentError.value_expected("utf8.offset", 2) + end + + defp utf8_offset([s | rest], state) do + if !is_binary(s) do + raise ArgumentError, + function_name: "utf8.offset", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + + n = check_integer(Enum.at(rest, 0), "utf8.offset", 2) + len = byte_size(s) + default_i = if n >= 0, do: 1, else: len + 1 + raw_i = Enum.at(rest, 1, default_i) + i = check_integer(raw_i, "utf8.offset", 3) + posi = u_posrelat(i, len) + + if posi < 1 or posi > len + 1 do + raise ArgumentError, + function_name: "utf8.offset", + arg_num: 3, + details: "position out of range" + end + + # Operate in 0-based byte index internally (`pos`); convert back at end. + pos = posi - 1 + result = do_offset(s, n, pos, len) + {result, state} + end + + defp do_offset(s, 0, pos, _len) do + pos = scan_back_to_char_start(s, pos) + [pos + 1] + end + + defp do_offset(s, n, pos, len) do + if pos < len and is_continuation_byte?(:binary.at(s, pos)) do + raise RuntimeError, value: "initial position is a continuation byte" + end + + # PUC decrements n by one before entering the advance loop, so + # `utf8.offset(s, 1, i)` returns position `i` itself (no advance). + cond do + n > 0 -> advance_forward(s, n - 1, pos, len) + n < 0 -> advance_backward(s, n, pos) + end + end + + defp advance_forward(_s, 0, pos, _len), do: [pos + 1] + defp advance_forward(_s, _n, pos, len) when pos >= len, do: [nil] + + defp advance_forward(s, n, pos, len) do + next_pos = skip_continuations(s, pos + 1, len) + advance_forward(s, n - 1, next_pos, len) + end + + defp advance_backward(_s, 0, pos), do: [pos + 1] + defp advance_backward(_s, _n, pos) when pos <= 0, do: [nil] + + defp advance_backward(s, n, pos) do + prev_pos = scan_back_to_char_start(s, pos - 1) + advance_backward(s, n + 1, prev_pos) + end + + defp scan_back_to_char_start(_s, pos) when pos <= 0, do: 0 + + defp scan_back_to_char_start(s, pos) do + if is_continuation_byte?(:binary.at(s, pos)) do + scan_back_to_char_start(s, pos - 1) + else + pos + end + end + + defp skip_continuations(_s, pos, len) when pos >= len, do: pos + + defp skip_continuations(s, pos, len) do + if is_continuation_byte?(:binary.at(s, pos)) do + skip_continuations(s, pos + 1, len) + else + pos + end + end + + # --------------------------------------------------------------------------- + # utf8.codes(s) + # --------------------------------------------------------------------------- + + defp utf8_codes([], _state) do + raise ArgumentError.value_expected("utf8.codes", 1) + end + + defp utf8_codes([s | _], state) when is_binary(s) do + iter = {:native_func, &codes_iter/2} + {[iter, s, 0], state} + end + + defp utf8_codes([v | _], _state) do + raise ArgumentError, + function_name: "utf8.codes", + arg_num: 1, + expected: "string", + got: Util.typeof(v) + end + + defp codes_iter([s, prev_pos], state) when is_binary(s) and is_integer(prev_pos) do + len = byte_size(s) + + pos = + cond do + prev_pos == 0 -> + 1 + + prev_pos >= len -> + len + 1 + + true -> + # Advance past the codepoint at prev_pos. Always at least +1 byte; + # skip any trailing continuation bytes. + p = prev_pos + 1 + skip_continuations_1based(s, p, len) + end + + if pos > len do + {[nil], state} + else + case decode_at(s, pos) do + {cp, _len} -> {[pos, cp], state} + :invalid -> raise RuntimeError, value: "invalid UTF-8 code" + end + end + end + + defp skip_continuations_1based(_s, pos, len) when pos > len, do: pos + + defp skip_continuations_1based(s, pos, len) do + if is_continuation_byte?(:binary.at(s, pos - 1)) do + skip_continuations_1based(s, pos + 1, len) + else + pos + end + end + + # --------------------------------------------------------------------------- + # Shared helpers + # --------------------------------------------------------------------------- + + defp check_integer(v, _fn, _arg) when is_integer(v), do: v + + defp check_integer(v, fn_name, arg) when is_float(v) do + int = trunc(v) + + if int == v do + int + else + raise ArgumentError, + function_name: fn_name, + arg_num: arg, + details: "number has no integer representation" + end + end + + defp check_integer(v, fn_name, arg) do + raise ArgumentError, + function_name: fn_name, + arg_num: arg, + expected: "number", + got: Util.typeof(v) + end + + # Lua 5.3 PUC u_posrelat: normalise a possibly-negative position to a + # 1-based byte index. Returns 0 when a negative position underflows the + # start of the string. + defp u_posrelat(pos, _len) when pos >= 0, do: pos + defp u_posrelat(pos, len) when -pos > len, do: 0 + defp u_posrelat(pos, len), do: len + pos + 1 + + defp is_continuation_byte?(byte), do: (byte &&& 0xC0) == 0x80 + + # Decode the UTF-8 sequence starting at 1-based position `pos` in `s`. + # Returns `{codepoint, byte_length}` or `:invalid` if the bytes at `pos` + # do not start a well-formed UTF-8 sequence (rejects overlong encodings + # and codepoints above 0x10FFFF). + defp decode_at(s, pos) when pos >= 1 and pos <= byte_size(s) do + slice_len = min(byte_size(s) - (pos - 1), 4) + decode_codepoint(:binary.part(s, pos - 1, slice_len)) + end + + defp decode_at(_, _), do: :invalid + + defp decode_codepoint(<>) when b1 < 0x80, do: {b1, 1} + + defp decode_codepoint(<>) when b1 >= 0xC2 and b1 <= 0xDF and (b2 &&& 0xC0) == 0x80 do + {(b1 &&& 0x1F) <<< 6 ||| (b2 &&& 0x3F), 2} + end + + defp decode_codepoint(<>) + when b1 >= 0xE0 and b1 <= 0xEF and (b2 &&& 0xC0) == 0x80 and (b3 &&& 0xC0) == 0x80 do + cp = (b1 &&& 0x0F) <<< 12 ||| (b2 &&& 0x3F) <<< 6 ||| (b3 &&& 0x3F) + if cp < 0x800, do: :invalid, else: {cp, 3} + end + + defp decode_codepoint(<>) + when b1 >= 0xF0 and b1 <= 0xF7 and (b2 &&& 0xC0) == 0x80 and (b3 &&& 0xC0) == 0x80 and (b4 &&& 0xC0) == 0x80 do + cp = + (b1 &&& 0x07) <<< 18 ||| (b2 &&& 0x3F) <<< 12 ||| (b3 &&& 0x3F) <<< 6 ||| (b4 &&& 0x3F) + + cond do + cp < 0x10000 -> :invalid + cp > @max_codepoint -> :invalid + true -> {cp, 4} + end + end + + defp decode_codepoint(_), do: :invalid +end diff --git a/test/lua/vm/arithmetic_test.exs b/test/lua/vm/arithmetic_test.exs index c8318627..5692f958 100644 --- a/test/lua/vm/arithmetic_test.exs +++ b/test/lua/vm/arithmetic_test.exs @@ -136,7 +136,7 @@ defmodule Lua.VM.ArithmeticTest do assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") state = State.new() - assert_raise RuntimeError, ~r/divide by zero/, fn -> + assert_raise RuntimeError, ~r/attempt to perform 'n\/\/0'/, fn -> VM.execute(proto, state) end end @@ -147,7 +147,7 @@ defmodule Lua.VM.ArithmeticTest do assert {:ok, proto} = Compiler.compile(ast, source: "test.lua") state = State.new() - assert_raise RuntimeError, ~r/modulo by zero/, fn -> + assert_raise RuntimeError, ~r/attempt to perform 'n%0'/, fn -> VM.execute(proto, state) end end @@ -251,7 +251,7 @@ defmodule Lua.VM.ArithmeticTest do state = Stdlib.install(State.new()) assert {:ok, [false, err], _state} = VM.execute(proto, state) assert is_binary(err) - assert err =~ "divide by zero" + assert err =~ "attempt to perform 'n//0'" end test "pcall catches comparison TypeError" do diff --git a/test/lua/vm/error_format_test.exs b/test/lua/vm/error_format_test.exs new file mode 100644 index 00000000..b70382b6 --- /dev/null +++ b/test/lua/vm/error_format_test.exs @@ -0,0 +1,90 @@ +defmodule Lua.VM.ErrorFormatTest do + use ExUnit.Case, async: true + + alias Lua.Compiler + alias Lua.Parser + alias Lua.VM + alias Lua.VM.ArgumentError, as: LuaArgumentError + alias Lua.VM.RuntimeError, as: LuaRuntimeError + alias Lua.VM.State + alias Lua.VM.Stdlib + alias Lua.VM.TypeError, as: LuaTypeError + + defp run(code) do + {:ok, ast} = Parser.parse(code) + {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + VM.execute(proto, state) + end + + # Pins the PUC-Lua-aligned error templates listed in issue #252. + # Each test guards one wording the official Lua 5.3 suite greps for via + # checkerror/checkmessage. Loosening any of these will silently break + # suite progress. + + describe "wrong number of arguments to 'X'" do + test "table.insert with too many args" do + assert_raise LuaRuntimeError, ~r/wrong number of arguments to 'insert'/, fn -> + run("return table.insert({}, 2, 3, 4)") + end + end + end + + describe "bad argument #N to 'X' (T expected, got T)" do + test "math.abs with string" do + assert_raise LuaArgumentError, + ~r/bad argument #1 to 'math\.abs' \(number expected, got string\)/, + fn -> run("return math.abs('foo')") end + end + + test "string.sub with non-integer index" do + assert_raise LuaArgumentError, + ~r/bad argument #2 to 'string\.sub' \(number expected\)/, + fn -> run("return string.sub('abc', 'x')") end + end + + test "table.insert with non-table first arg" do + assert_raise LuaArgumentError, + ~r/bad argument #1 to 'table\.insert' \(table expected, got number\)/, + fn -> run("return table.insert(1, 2)") end + end + end + + describe "value expected" do + test "math.abs with no args" do + assert_raise LuaArgumentError, + ~r/bad argument #1 to 'math\.abs' \(value expected\)/, + fn -> run("return math.abs()") end + end + end + + describe "attempt to perform 'n//0'" do + test "integer floor-divide by zero" do + assert_raise LuaRuntimeError, ~r/attempt to perform 'n\/\/0'/, fn -> + run("return 1 // 0") + end + end + end + + describe "attempt to perform 'n%0'" do + test "integer modulo by zero" do + assert_raise LuaRuntimeError, ~r/attempt to perform 'n%0'/, fn -> + run("return 1 % 0") + end + end + end + + describe "number has no integer representation" do + test "bitwise on a non-representable float" do + assert_raise LuaTypeError, ~r/has no integer representation/, fn -> + run("return (2.5) | 0") + end + end + + test "bitwise on math.huge surfaces the same template" do + assert_raise LuaTypeError, ~r/has no integer representation/, fn -> + run("return math.huge << 1") + end + end + end +end diff --git a/test/lua/vm/float_div_zero_test.exs b/test/lua/vm/float_div_zero_test.exs index 6d33fdf8..9e1ec2f2 100644 --- a/test/lua/vm/float_div_zero_test.exs +++ b/test/lua/vm/float_div_zero_test.exs @@ -80,13 +80,13 @@ defmodule Lua.VM.FloatDivZeroTest do describe "floor div and modulo with integer-zero divisor still raise" do test "1 // 0 raises" do - assert_raise Lua.RuntimeException, ~r/divide by zero/, fn -> + assert_raise Lua.RuntimeException, ~r/attempt to perform 'n\/\/0'/, fn -> Lua.eval!(Lua.new(), "return 1 // 0") end end test "1 % 0 raises" do - assert_raise Lua.RuntimeException, ~r/modulo by zero/, fn -> + assert_raise Lua.RuntimeException, ~r/attempt to perform 'n%0'/, fn -> Lua.eval!(Lua.new(), "return 1 % 0") end end @@ -150,7 +150,7 @@ defmodule Lua.VM.FloatDivZeroTest do end test "1.0 // 0 still raises — integer divisor" do - assert_raise Lua.RuntimeException, ~r/divide by zero/, fn -> + assert_raise Lua.RuntimeException, ~r/attempt to perform 'n\/\/0'/, fn -> Lua.eval!(Lua.new(), "return 1.0 // 0") end end @@ -160,7 +160,7 @@ defmodule Lua.VM.FloatDivZeroTest do end test "1.0 % 0 still raises — integer divisor" do - assert_raise Lua.RuntimeException, ~r/modulo by zero/, fn -> + assert_raise Lua.RuntimeException, ~r/attempt to perform 'n%0'/, fn -> Lua.eval!(Lua.new(), "return 1.0 % 0") end end diff --git a/test/lua/vm/stdlib/utf8_test.exs b/test/lua/vm/stdlib/utf8_test.exs new file mode 100644 index 00000000..9de48abe --- /dev/null +++ b/test/lua/vm/stdlib/utf8_test.exs @@ -0,0 +1,222 @@ +defmodule Lua.VM.Stdlib.Utf8Test do + use ExUnit.Case, async: true + + alias Lua.Compiler + alias Lua.Parser + alias Lua.VM + alias Lua.VM.ArgumentError, as: LuaArgumentError + alias Lua.VM.RuntimeError, as: LuaRuntimeError + alias Lua.VM.State + alias Lua.VM.Stdlib + + defp run(code) do + {:ok, ast} = Parser.parse(code) + {:ok, proto} = Compiler.compile(ast, source: "test.lua") + state = Stdlib.install(State.new()) + VM.execute(proto, state) + end + + describe "utf8.char" do + test "encodes single-byte codepoints" do + assert {:ok, ["abc"], _} = run("return utf8.char(97, 98, 99)") + end + + test "encodes empty arg list to empty string" do + assert {:ok, [""], _} = run("return utf8.char()") + end + + test "encodes 2/3/4-byte sequences" do + # 0x80, 0x7FF, 0x800, 0xFFFF, 0x10000, 0x10FFFF — boundaries + assert {:ok, [bin], _} = + run("return utf8.char(0x80, 0x7FF, 0x800, 0xFFFF, 0x10000, 0x10FFFF)") + + assert bin == + <<0xC2, 0x80, 0xDF, 0xBF, 0xE0, 0xA0, 0x80, 0xEF, 0xBF, 0xBF, 0xF0, 0x90, 0x80, 0x80, 0xF4, 0x8F, 0xBF, + 0xBF>> + end + + test "raises value out of range above 0x10FFFF" do + assert_raise LuaArgumentError, ~r/bad argument #1 to 'utf8\.char' \(value out of range\)/, fn -> + run("return utf8.char(0x110000)") + end + end + + test "raises on non-integer arg" do + assert_raise LuaArgumentError, + ~r/bad argument #1 to 'utf8\.char' \(number expected, got string\)/, + fn -> + run("return utf8.char('a')") + end + end + end + + describe "utf8.codepoint" do + test "default i,j returns first codepoint" do + assert {:ok, [97], _} = run("return utf8.codepoint('abc')") + end + + test "returns range of codepoints" do + assert {:ok, [97, 98, 99], _} = run("return utf8.codepoint('abc', 1, 3)") + end + + test "multi-byte sequence" do + # 汉 = 0x6C49, 字 = 0x5B57 + assert {:ok, [0x6C49, 0x5B57], _} = run("return utf8.codepoint('汉字', 1, -1)") + end + + test "raises on invalid UTF-8 byte" do + assert_raise LuaRuntimeError, ~r/invalid UTF-8 code/, fn -> + run("return utf8.codepoint('\\xff')") + end + end + + test "raises on i out of range (below)" do + assert_raise LuaArgumentError, ~r/bad argument #2 to 'utf8\.codepoint' \(out of range\)/, fn -> + run("local s = 'abc'; return utf8.codepoint(s, -(#s + 1))") + end + end + + test "raises on j out of range (above)" do + assert_raise LuaArgumentError, ~r/bad argument #3 to 'utf8\.codepoint' \(out of range\)/, fn -> + run("local s = 'abc'; return utf8.codepoint(s, 1, #s + 1)") + end + end + end + + describe "utf8.len" do + test "ASCII string length is byte count" do + assert {:ok, [11], _} = run("return utf8.len('hello World')") + end + + test "counts multi-byte characters as 1 each" do + assert {:ok, [2], _} = run("return utf8.len('汉字')") + end + + test "empty string is length 0" do + assert {:ok, [0], _} = run("return utf8.len('')") + end + + test "returns nil + position on invalid sequence" do + # 'abc' + \xE3 starts an unfinished 3-byte sequence at byte 4 + assert {:ok, [nil, 4], _} = run("return utf8.len('abc\\xE3def')") + end + + test "range arguments" do + assert {:ok, [1], _} = run("return utf8.len('abc', 2, 2)") + end + end + + describe "utf8.offset" do + test "returns byte position of n-th character" do + # 汉字 = 6 bytes (3 + 3). 1st char at byte 1, 2nd at byte 4. + assert {:ok, [1], _} = run("return utf8.offset('汉字', 1)") + assert {:ok, [4], _} = run("return utf8.offset('汉字', 2)") + end + + test "n=0 returns start of character containing byte i" do + # 4-byte char 𦧺 starts at byte 1. From any byte 1..4, n=0 returns 1. + assert {:ok, [1], _} = run("return utf8.offset('𦧺', 0, 1)") + assert {:ok, [1], _} = run("return utf8.offset('𦧺', 0, 4)") + end + + test "negative n counts backward" do + # "abc": 3 ASCII chars. n=-1 from end position (4) = last char position (3). + assert {:ok, [3], _} = run("return utf8.offset('abc', -1)") + end + + test "returns nil when past end" do + assert {:ok, [nil], _} = run("return utf8.offset('alo', 5)") + assert {:ok, [nil], _} = run("return utf8.offset('alo', -4)") + end + + test "raises position out of range" do + assert_raise LuaArgumentError, + ~r/bad argument #3 to 'utf8\.offset' \(position out of range\)/, + fn -> + run("return utf8.offset('abc', 1, 5)") + end + end + + test "raises position out of range (negative)" do + assert_raise LuaArgumentError, + ~r/bad argument #3 to 'utf8\.offset' \(position out of range\)/, + fn -> + run("return utf8.offset('abc', 1, -4)") + end + end + + test "raises on continuation byte at non-zero n" do + # 𦧺 is 4 bytes; byte 2 is a continuation byte + assert_raise LuaRuntimeError, ~r/initial position is a continuation byte/, fn -> + run("return utf8.offset('𦧺', 1, 2)") + end + end + end + + describe "utf8.codes" do + test "iterates over ASCII codepoints" do + code = """ + local t = {} + for p, c in utf8.codes('abc') do + t[#t + 1] = p + t[#t + 1] = c + end + return t[1], t[2], t[3], t[4], t[5], t[6] + """ + + assert {:ok, [1, 97, 2, 98, 3, 99], _} = run(code) + end + + test "iterates over multi-byte codepoints" do + code = """ + local t = {} + for p, c in utf8.codes('汉字') do + t[#t + 1] = c + end + return t[1], t[2] + """ + + assert {:ok, [0x6C49, 0x5B57], _} = run(code) + end + + test "raises on invalid sequence mid-iteration" do + code = """ + for p, c in utf8.codes('ab\\xff') do end + """ + + assert_raise LuaRuntimeError, ~r/invalid UTF-8 code/, fn -> + run(code) + end + end + end + + describe "utf8.charpattern" do + test "is the Lua 5.3 canonical pattern string" do + assert {:ok, [pat], _} = run("return utf8.charpattern") + assert pat == <<"[\0-\x7F\xC2-\xFD][\x80-\xBF]*">> + end + + test "matches one UTF-8 sequence at a time via gmatch" do + code = """ + local t = {} + for c in string.gmatch('汉a字', utf8.charpattern) do + t[#t + 1] = c + end + return t[1], t[2], t[3] + """ + + assert {:ok, ["汉", "a", "字"], _} = run(code) + end + end + + describe "require('utf8')" do + test "resolves to the utf8 library table" do + code = """ + local m = require('utf8') + return m == utf8, m.char(97) + """ + + assert {:ok, [true, "a"], _} = run(code) + end + end +end diff --git a/test/lua53_skips.exs b/test/lua53_skips.exs index b5db2f68..f27d2513 100644 --- a/test/lua53_skips.exs +++ b/test/lua53_skips.exs @@ -170,13 +170,5 @@ reason: "string.rep with large counts and checkerror format mismatches; pending finer triage", issue: nil } - ], - "utf8.lua" => [ - %{ - lines: :all, - category: :unimplemented, - reason: "utf8 standard library not implemented", - issue: nil - } ] } From c7af25d48b080cc45d2bab798e8f663522ab889b Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Thu, 28 May 2026 06:59:04 -0400 Subject: [PATCH 2/2] stdlib(utf8): accept integer-valued floats; rename continuation predicate - utf8.char now routes through check_integer, so 97.0 succeeds and 97.5 raises the PUC-aligned "number has no integer representation" message (previously fell through to "number expected, got number"). - Rename is_continuation_byte? -> continuation_byte? to match the repo convention that reserves is_ for guards. - Replace `if !is_binary(s)` with function-head `when not is_binary(s)` guards across the four entry points. --- lib/lua/vm/stdlib/utf8.ex | 92 ++++++++++++++------------------ test/lua/vm/stdlib/utf8_test.exs | 12 +++++ 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/lib/lua/vm/stdlib/utf8.ex b/lib/lua/vm/stdlib/utf8.ex index 405137ea..114c2880 100644 --- a/lib/lua/vm/stdlib/utf8.ex +++ b/lib/lua/vm/stdlib/utf8.ex @@ -63,23 +63,17 @@ defmodule Lua.VM.Stdlib.Utf8 do {[IO.iodata_to_binary(iolist)], state} end - defp encode_one(cp, _idx) when is_integer(cp) and cp >= 0 and cp <= @max_codepoint do - encode_codepoint(cp) - end + defp encode_one(cp, idx) do + cp = check_integer(cp, "utf8.char", idx) - defp encode_one(cp, idx) when is_integer(cp) do - raise ArgumentError, - function_name: "utf8.char", - arg_num: idx, - details: "value out of range" - end - - defp encode_one(v, idx) do - raise ArgumentError, - function_name: "utf8.char", - arg_num: idx, - expected: "number", - got: Util.typeof(v) + if cp >= 0 and cp <= @max_codepoint do + encode_codepoint(cp) + else + raise ArgumentError, + function_name: "utf8.char", + arg_num: idx, + details: "value out of range" + end end defp encode_codepoint(cp) when cp < 0x80, do: <> @@ -104,15 +98,15 @@ defmodule Lua.VM.Stdlib.Utf8 do raise ArgumentError.value_expected("utf8.codepoint", 1) end - defp utf8_codepoint([s | rest], state) do - if !is_binary(s) do - raise ArgumentError, - function_name: "utf8.codepoint", - arg_num: 1, - expected: "string", - got: Util.typeof(s) - end + defp utf8_codepoint([s | _], _state) when not is_binary(s) do + raise ArgumentError, + function_name: "utf8.codepoint", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + defp utf8_codepoint([s | rest], state) do len = byte_size(s) i = check_integer(Enum.at(rest, 0, 1), "utf8.codepoint", 2) j = check_integer(Enum.at(rest, 1, i), "utf8.codepoint", 3) @@ -156,15 +150,15 @@ defmodule Lua.VM.Stdlib.Utf8 do raise ArgumentError.value_expected("utf8.len", 1) end - defp utf8_len([s | rest], state) do - if !is_binary(s) do - raise ArgumentError, - function_name: "utf8.len", - arg_num: 1, - expected: "string", - got: Util.typeof(s) - end + defp utf8_len([s | _], _state) when not is_binary(s) do + raise ArgumentError, + function_name: "utf8.len", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + defp utf8_len([s | rest], state) do len = byte_size(s) i = check_integer(Enum.at(rest, 0, 1), "utf8.len", 2) j = check_integer(Enum.at(rest, 1, -1), "utf8.len", 3) @@ -206,27 +200,19 @@ defmodule Lua.VM.Stdlib.Utf8 do raise ArgumentError.value_expected("utf8.offset", 1) end - defp utf8_offset([s], _state) do - if !is_binary(s) do - raise ArgumentError, - function_name: "utf8.offset", - arg_num: 1, - expected: "string", - got: Util.typeof(s) - end + defp utf8_offset([s | _], _state) when not is_binary(s) do + raise ArgumentError, + function_name: "utf8.offset", + arg_num: 1, + expected: "string", + got: Util.typeof(s) + end + defp utf8_offset([s], _state) when is_binary(s) do raise ArgumentError.value_expected("utf8.offset", 2) end defp utf8_offset([s | rest], state) do - if !is_binary(s) do - raise ArgumentError, - function_name: "utf8.offset", - arg_num: 1, - expected: "string", - got: Util.typeof(s) - end - n = check_integer(Enum.at(rest, 0), "utf8.offset", 2) len = byte_size(s) default_i = if n >= 0, do: 1, else: len + 1 @@ -253,7 +239,7 @@ defmodule Lua.VM.Stdlib.Utf8 do end defp do_offset(s, n, pos, len) do - if pos < len and is_continuation_byte?(:binary.at(s, pos)) do + if pos < len and continuation_byte?(:binary.at(s, pos)) do raise RuntimeError, value: "initial position is a continuation byte" end @@ -284,7 +270,7 @@ defmodule Lua.VM.Stdlib.Utf8 do defp scan_back_to_char_start(_s, pos) when pos <= 0, do: 0 defp scan_back_to_char_start(s, pos) do - if is_continuation_byte?(:binary.at(s, pos)) do + if continuation_byte?(:binary.at(s, pos)) do scan_back_to_char_start(s, pos - 1) else pos @@ -294,7 +280,7 @@ defmodule Lua.VM.Stdlib.Utf8 do defp skip_continuations(_s, pos, len) when pos >= len, do: pos defp skip_continuations(s, pos, len) do - if is_continuation_byte?(:binary.at(s, pos)) do + if continuation_byte?(:binary.at(s, pos)) do skip_continuations(s, pos + 1, len) else pos @@ -353,7 +339,7 @@ defmodule Lua.VM.Stdlib.Utf8 do defp skip_continuations_1based(_s, pos, len) when pos > len, do: pos defp skip_continuations_1based(s, pos, len) do - if is_continuation_byte?(:binary.at(s, pos - 1)) do + if continuation_byte?(:binary.at(s, pos - 1)) do skip_continuations_1based(s, pos + 1, len) else pos @@ -394,7 +380,7 @@ defmodule Lua.VM.Stdlib.Utf8 do defp u_posrelat(pos, len) when -pos > len, do: 0 defp u_posrelat(pos, len), do: len + pos + 1 - defp is_continuation_byte?(byte), do: (byte &&& 0xC0) == 0x80 + defp continuation_byte?(byte), do: (byte &&& 0xC0) == 0x80 # Decode the UTF-8 sequence starting at 1-based position `pos` in `s`. # Returns `{codepoint, byte_length}` or `:invalid` if the bytes at `pos` diff --git a/test/lua/vm/stdlib/utf8_test.exs b/test/lua/vm/stdlib/utf8_test.exs index 9de48abe..f4a3d5f0 100644 --- a/test/lua/vm/stdlib/utf8_test.exs +++ b/test/lua/vm/stdlib/utf8_test.exs @@ -48,6 +48,18 @@ defmodule Lua.VM.Stdlib.Utf8Test do run("return utf8.char('a')") end end + + test "accepts floats with integer representation" do + assert {:ok, ["a"], _} = run("return utf8.char(97.0)") + end + + test "raises on float with no integer representation" do + assert_raise LuaArgumentError, + ~r/bad argument #1 to 'utf8\.char' \(number has no integer representation\)/, + fn -> + run("return utf8.char(97.5)") + end + end end describe "utf8.codepoint" do