Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 69 additions & 16 deletions lib/lua/parser.ex
Original file line number Diff line number Diff line change
Expand Up @@ -960,8 +960,8 @@ defmodule Lua.Parser do
defp parse_paren_expr([{:delimiter, :lparen, pos} | rest]) do
case parse_expr(rest) do
{:ok, expr, rest2} ->
case expect(rest2, :delimiter, :rparen) do
{:ok, _, rest3} ->
case expect_closing(rest2, :rparen, :lparen, pos) do
{:ok, rest3} ->
{:ok, wrap_paren(expr, pos), rest3}

{:error, reason} ->
Expand All @@ -985,8 +985,8 @@ defmodule Lua.Parser do
defp parse_table([{:delimiter, :lbrace, pos} | rest]) do
case parse_table_fields(rest, []) do
{:ok, fields, rest2} ->
case expect(rest2, :delimiter, :rbrace) do
{:ok, _, rest3} ->
case expect_closing(rest2, :rbrace, :lbrace, pos) do
{:ok, rest3} ->
{:ok, %Expr.Table{fields: Enum.reverse(fields), meta: Meta.new(pos)}, rest3}

{:error, reason} ->
Expand Down Expand Up @@ -1132,16 +1132,16 @@ defmodule Lua.Parser do
end

# Parse function call arguments: (args)
defp parse_call_args([{:delimiter, :lparen, _} | rest]) do
parse_expr_list_until(rest, :rparen)
defp parse_call_args([{:delimiter, :lparen, open_pos} | rest]) do
parse_expr_list_until(rest, :rparen, :lparen, open_pos)
end

# Parse indexing: [key]
defp parse_index([{:delimiter, :lbracket, _} | rest]) do
defp parse_index([{:delimiter, :lbracket, open_pos} | rest]) do
case parse_expr(rest) do
{:ok, key, rest2} ->
case expect(rest2, :delimiter, :rbracket) do
{:ok, _, rest3} ->
case expect_closing(rest2, :rbracket, :lbracket, open_pos) do
{:ok, rest3} ->
{:ok, key, rest3}

{:error, reason} ->
Expand Down Expand Up @@ -1177,15 +1177,43 @@ defmodule Lua.Parser do
end

{:error, reason} ->
if acc == [] do
{:error, reason}
else
{:ok, Enum.reverse(acc), tokens}
# Recover (treat the list as ended) ONLY when the failed sub-parse
# made no progress — i.e. the next token simply cannot begin an
# expression, so the caller's terminator check should produce the
# error. If the sub-parse *committed* (consumed input before
# failing), that deeper error is the real one and must propagate;
# swallowing it here surfaces a misleading "expected <terminator>"
# at the list boundary instead of pointing at the actual mistake.
cond do
acc == [] -> {:error, reason}
committed?(reason, tokens) -> {:error, reason}
true -> {:ok, Enum.reverse(acc), tokens}
end
end
end

defp parse_expr_list_until(tokens, terminator) do
# A sub-parse "committed" if it consumed input before failing. We detect
# this by comparing the error's byte offset against the offset of the
# token the sub-parse started on: an error positioned strictly past the
# start means tokens were consumed. `:unexpected_end` (EOF) is always
# committed — running out of input mid-list is a real error, never a
# clean list terminator. A missing position falls back to "not
# committed", preserving the previous recovery behaviour.
defp committed?({:unexpected_end, _message, _pos}, _tokens), do: true

defp committed?(reason, tokens) do
with %{byte_offset: error_offset} <- error_position(reason),
{_, _, %{byte_offset: start_offset}} <- peek(tokens) do
error_offset > start_offset
else
_ -> false
end
end

defp error_position({:unexpected_token, _type, pos, _message}), do: pos
defp error_position(_), do: nil

defp parse_expr_list_until(tokens, terminator, delimiter, open_pos) do
tokens = skip_comments(tokens)

case peek(tokens) do
Expand All @@ -1200,8 +1228,8 @@ defmodule Lua.Parser do
# the terminator (e.g. `f(1, 2 -- note\n)`).
rest = skip_comments(rest)

case expect(rest, :delimiter, terminator) do
{:ok, _, rest2} ->
case expect_closing(rest, terminator, delimiter, open_pos) do
{:ok, rest2} ->
{:ok, exprs, rest2}

{:error, reason} ->
Expand All @@ -1214,6 +1242,27 @@ defmodule Lua.Parser do
end
end

# Expect a closing delimiter. When the stream is exhausted (EOF), the
# opener was never closed — report it against the opening position (with a
# "add a closing X at line N" suggestion) rather than a bare "expected X,
# got eof" pinned to the end of the file. A non-EOF mismatch keeps the
# specific "expected X, got <token>" error pointing at the offending token.
defp expect_closing(tokens, terminator, delimiter, open_pos) do
case expect(tokens, :delimiter, terminator) do
{:ok, _, rest} ->
{:ok, rest}

{:error, {:unexpected_token, :eof, _close_pos, _msg}} ->
{:error, {:unclosed_delimiter, delimiter, open_pos}}

{:error, {:unexpected_end, _msg, _close_pos}} ->
{:error, {:unclosed_delimiter, delimiter, open_pos}}

{:error, reason} ->
{:error, reason}
end
end

# Token manipulation helpers

defp peek([token | _]), do: token
Expand Down Expand Up @@ -1312,6 +1361,10 @@ defmodule Lua.Parser do
Error.new(:unexpected_token, message, pos, suggestion: suggest_for_token_error(type, message))
end

defp convert_error({:unclosed_delimiter, delimiter, open_pos}, _code) do
Error.unclosed_delimiter(delimiter, open_pos)
end

defp convert_error({:unexpected_end, message, pos}, _code) do
Error.new(:unexpected_end, message, pos,
suggestion: """
Expand Down
111 changes: 111 additions & 0 deletions test/lua/parser/error_position_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
defmodule Lua.Parser.ErrorPositionTest do
@moduledoc """
Pins the *position* of syntax errors, not just their detection.

A recursive-descent parser can detect a syntax error correctly yet report
it at the wrong place: when a deeply-nested sub-parse fails, a shallower
recovery point may swallow the real error and substitute a generic
"expected <terminator>" pinned to the list boundary. These tests lock the
reported line to the actual offending token.

The golden corpus is CI-deterministic. The differential test cross-checks
our reported line against reference Lua (`luac -p`) and is skipped when no
`luac` is on PATH.
"""
use ExUnit.Case, async: true

alias Lua.Parser

# {name, source, expected_line, message_substring, reference}
#
# `reference` says how our line relates to PUC-Lua's:
# :match -> there is a concrete offending token; we agree with luac.
# :opener -> the construct runs off the end of input; we deliberately
# point at the *opening* delimiter (luac points at <eof>).
@corpus [
# The reported regression: a deep error inside the 5th argument (a
# function literal) of a method call. Before the fix this blamed the
# `function` keyword on the call line; it must now point at the `end`
# where an expression was expected.
{"deep error in 5th call argument", ~s|w:step("a", "b", "c", {x = 1}, function()\n bar(\nend)\n|, 3,
"Expected expression", :match},

# Deep error inside an argument fails immediately on a bad token.
{"deep error in a table field value", "t = {\n a = 1,\n b = foo(,\n}\n", 3, "Expected expression", :match},

# Deep error inside a return expression list.
{"deep error in a return list", "function f()\n return 1, 2, g(\nend\n", 3, "Expected expression", :match},

# Stray statement after closed nested calls — error at the garbage line.
{"stray statement after nested calls", "outer(mid(inner(\n)))\n.. bad\n", 3, "bare", :match},

# We must NOT over-propagate: a clean list end followed by a stray token
# still reports the stray token at the boundary, not a deeper error.
{"stray token after complete arg list", "print(1, 2 3)\n", 1, "Expected", :match},

# Genuinely-unclosed constructs at EOF point at the opener line.
{"unclosed call across lines", "print(1, 2, 3\n", 1, "Unclosed", :opener},
{"unclosed index across lines", "y = t[\n 1\n", 1, "Unclosed", :opener},
{"unclosed paren across lines", "z = (\n 1 + 2\n", 1, "Unclosed", :opener}
]

describe "golden error positions" do
for {name, source, expected_line, substring, _ref} <- @corpus do
test name do
assert {:error, msg} = Parser.parse(unquote(source))
clean = strip_ansi(msg)

assert clean =~ "at line #{unquote(expected_line)},",
"expected error at line #{unquote(expected_line)}, got:\n#{clean}"

assert clean =~ unquote(substring)
end
end
end

describe "differential vs. reference Lua (luac -p)" do
@describetag :differential

setup do
case System.find_executable("luac") do
nil -> {:skip, "luac not found on PATH"}
path -> {:ok, luac: path}
end
end

for {name, source, _expected_line, _substring, :match} <- @corpus do
test "agrees with luac on: #{name}", %{luac: luac} do
assert {:error, msg} = Parser.parse(unquote(source))
ours = error_line(strip_ansi(msg))

assert is_integer(ours)
assert reference_line(luac, unquote(source)) == ours
end
end
end

defp strip_ansi(string), do: String.replace(string, ~r/\e\[[0-9;]*m/, "")

defp error_line(clean) do
case Regex.run(~r/at line (\d+)/, clean) do
[_, n] -> String.to_integer(n)
_ -> nil
end
end

defp reference_line(luac, source) do
path = Path.join(System.tmp_dir!(), "lua_error_position_#{:erlang.unique_integer([:positive])}.lua")
File.write!(path, source)

try do
{out, _status} = System.cmd(luac, ["-p", path], stderr_to_stdout: true)

case Regex.run(~r/:(\d+):/, out) do
[_, n] -> String.to_integer(n)
_ -> nil
end
after
File.rm(path)
end
end
end
5 changes: 4 additions & 1 deletion test/test_helper.exs
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
ExUnit.start(exclude: [:slow])
# `:differential` tests shell out to a reference `luac` and compare exact
# line numbers; they are environment-dependent (luac version / availability)
# and opt-in. Run them with `mix test --include differential`.
ExUnit.start(exclude: [:slow, :differential])
Loading