Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions lib/lua/parser.ex
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,7 @@ defmodule Lua.Parser do

# Parse table constructor: { fields }
defp parse_table([{:delimiter, :lbrace, pos} | rest]) do
case parse_table_fields(rest, []) do
case parse_table_fields(rest, [], pos) do
{:ok, fields, rest2} ->
case expect_closing(rest2, :rbrace, :lbrace, pos) do
{:ok, rest3} ->
Expand All @@ -1030,7 +1030,7 @@ defmodule Lua.Parser do
end
end

defp parse_table_fields(tokens, acc) do
defp parse_table_fields(tokens, acc, open_pos) do
# Skip leading comments (and any orphaned trailing comments from a
# previous field) before deciding whether we've hit the terminator
# or another field.
Expand All @@ -1040,6 +1040,12 @@ defmodule Lua.Parser do
{:delimiter, :rbrace, _} ->
{:ok, acc, tokens}

# The constructor ran off the end of input before a closing '}'.
# Point at the opening brace (like '(' and '[') rather than emitting
# a bare "expected ',' or '}', got eof" pinned to the end of the file.
{:eof, _} ->
{:error, {:unclosed_delimiter, :lbrace, open_pos}}

_ ->
case parse_table_field(tokens) do
{:ok, field, rest} ->
Expand All @@ -1050,15 +1056,20 @@ defmodule Lua.Parser do
case peek(rest) do
{:delimiter, :comma, _} ->
{_, rest2} = consume(rest)
parse_table_fields(rest2, [field | acc])
parse_table_fields(rest2, [field | acc], open_pos)

{:delimiter, :semicolon, _} ->
{_, rest2} = consume(rest)
parse_table_fields(rest2, [field | acc])
parse_table_fields(rest2, [field | acc], open_pos)

{:delimiter, :rbrace, _} ->
{:ok, [field | acc], rest}

# Field parsed cleanly but the stream ended before a separator
# or '}'. Genuinely unclosed — blame the opening brace.
{:eof, _} ->
{:error, {:unclosed_delimiter, :lbrace, open_pos}}

_ ->
unexpected_token_error(rest, "Expected ',' or '}' in table")
end
Expand Down Expand Up @@ -1253,6 +1264,12 @@ defmodule Lua.Parser do
{_, rest} = consume(tokens)
{:ok, [], rest}

# Opened and ran straight off the end of input (e.g. `f(` at EOF).
# Blame the opener, matching the args-then-EOF case below, instead
# of a bare "Expected expression" pinned to the end of the file.
{:eof, _} ->
{:error, {:unclosed_delimiter, delimiter, open_pos}}

_ ->
case parse_expr_list(tokens) do
{:ok, exprs, rest} ->
Expand Down
221 changes: 147 additions & 74 deletions test/lua/parser/error_position_test.exs
Original file line number Diff line number Diff line change
@@ -1,111 +1,184 @@
defmodule Lua.Parser.ErrorPositionTest do
@moduledoc """
Pins the *position* of syntax errors, not just their detection.
Pins the *position* of syntax errors — line and column — not just their
detection.

A recursive-descent parser can detect a syntax error correctly yet report
it at the wrong place: when a deeply-nested sub-parse fails, a shallower
recovery point may swallow the real error and substitute a generic
"expected <terminator>" pinned to the list boundary. These tests lock the
reported line to the actual offending token.

The golden corpus is CI-deterministic. The differential test cross-checks
our reported line against reference Lua (`luac -p`) and is skipped when no
`luac` is on PATH.
reported line and column to the place our convention says they belong.

The convention these tests encode (our bar, not any reference compiler's):

* A *concrete offending token* mid-stream is blamed at that token — never
at a shallower recovery boundary. A deep error inside the Nth argument
of a call wins over the outer call's terminator.
* A construct that *runs off the end of input* (`(`, `[`, `{`, a string,
a long string/comment) is blamed at its *opening* delimiter, with an
"add a closing X" suggestion — not at a bare <eof> pinned to the end of
the file.
* A block keyword body (`function`/`if`/`while`/`for`/`do`) that reaches
<eof> without `end` is reported where `end` was expected.

Every entry asserts both APIs: `parse_structured/1` for the exact
`{line, column}`, and `parse/1` for the rendered `at line L, column C`
banner plus a substring of the message it pins.
"""
use ExUnit.Case, async: true

alias Lua.Parser
alias Lua.Parser.Error

# {name, source, expected_line, message_substring, reference}
# {name, source, line, column, message_substring}
#
# `reference` says how our line relates to PUC-Lua's:
# :match -> there is a concrete offending token; we agree with luac.
# :opener -> the construct runs off the end of input; we deliberately
# point at the *opening* delimiter (luac points at <eof>).
@corpus [
# The reported regression: a deep error inside the 5th argument (a
# function literal) of a method call. Before the fix this blamed the
# `function` keyword on the call line; it must now point at the `end`
# where an expression was expected.
{"deep error in 5th call argument", ~s|w:step("a", "b", "c", {x = 1}, function()\n bar(\nend)\n|, 3,
"Expected expression", :match},

# Deep error inside an argument fails immediately on a bad token.
{"deep error in a table field value", "t = {\n a = 1,\n b = foo(,\n}\n", 3, "Expected expression", :match},

# Deep error inside a return expression list.
{"deep error in a return list", "function f()\n return 1, 2, g(\nend\n", 3, "Expected expression", :match},

# Stray statement after closed nested calls — error at the garbage line.
{"stray statement after nested calls", "outer(mid(inner(\n)))\n.. bad\n", 3, "bare", :match},

# We must NOT over-propagate: a clean list end followed by a stray token
# still reports the stray token at the boundary, not a deeper error.
{"stray token after complete arg list", "print(1, 2 3)\n", 1, "Expected", :match},

# Genuinely-unclosed constructs at EOF point at the opener line.
{"unclosed call across lines", "print(1, 2, 3\n", 1, "Unclosed", :opener},
{"unclosed index across lines", "y = t[\n 1\n", 1, "Unclosed", :opener},
{"unclosed paren across lines", "z = (\n 1 + 2\n", 1, "Unclosed", :opener}
# Grouped by the shape of the failure. Columns are 1-indexed and point at
# the exact token described above.

@opener_errors [
# Bracket delimiters that run off the end of input are blamed at the
# opener, uniformly across (), [] and {}.
{"unclosed call across lines", "print(1, 2, 3\n", 1, 6, "Unclosed opening parenthesis"},
{"unclosed index across lines", "y = t[\n 1\n", 1, 6, "Unclosed opening bracket"},
{"unclosed paren across lines", "z = (\n 1 + 2\n", 1, 5, "Unclosed opening parenthesis"},
{"unclosed table across lines", "t = {\n a = 1\n", 1, 5, "Unclosed opening brace"},

# A delimiter opened immediately before <eof> still blames the opener,
# not a bare "Expected expression" at the end of the file.
{"empty call hits end of input", "g(\n", 1, 2, "Unclosed opening parenthesis"},

# The innermost unclosed delimiter wins: table -> function -> call, the
# call's '(' on line 3 is the deepest opener.
{"nested table/function/call all unclosed", "t = {\n f = function()\n g(\n", 3, 6, "Unclosed opening parenthesis"}
]

describe "golden error positions" do
for {name, source, expected_line, substring, _ref} <- @corpus do
test name do
assert {:error, msg} = Parser.parse(unquote(source))
clean = strip_ansi(msg)
@lexer_errors [
{"unterminated string", "x = \"abc\n", 1, 9, "Unclosed string literal"},
{"unterminated long string", "x = [[abc\n", 2, 1, "Unclosed long string"},
{"unterminated long comment", "--[[ comment\nx = 1\n", 3, 1, "Unclosed multi-line comment"}
]

assert clean =~ "at line #{unquote(expected_line)},",
"expected error at line #{unquote(expected_line)}, got:\n#{clean}"
@missing_end_errors [
# Block bodies that reach <eof> without `end` are reported where `end`
# was expected (the line past the body), not at the opening keyword.
{"function body missing end", "function f()\n return 1\n", 3, 1, "Expected :keyword::end"},
{"if-then body missing end", "if x then\n y = 1\n", 3, 1, "Expected :keyword::end"},
{"while-do body missing end", "while c do\n y = 1\n", 3, 1, "Expected :keyword::end"},
{"do block missing end", "do\n y = 1\n", 3, 1, "Expected :keyword::end"},
{"for-do body missing end", "for i = 1, 2 do\n y = 1\n", 3, 1, "Expected :keyword::end"}
]

assert clean =~ unquote(substring)
@offending_token_errors [
# A concrete bad token is blamed exactly where it sits.
{"stray token after complete arg list", "print(1, 2 3)\n", 1, 12, "Expected :delimiter::rparen"},
{"empty argument before comma", "f(,)\n", 1, 3, "Expected expression"},
{"empty return expression", "function f()\n return ,\nend\n", 2, 10, "Expected expression"},
{"assignment with no right-hand side", "x = \n", 2, 1, "Expected expression"},
{"binary operator with no right operand", "y = 1 + \n", 2, 1, "Expected expression"},
{"unary minus with no operand", "y = -\n", 2, 1, "Expected expression"},
{"'not' with no operand", "y = not\n", 2, 1, "Expected expression"},
# The motivating report: `function` as a statement needs a name, so the
# '(' is the genuine first error — line 1, not the call on line 2.
{"function statement missing name", "function() do\n foo.bar(\nend\n", 1, 9, "Expected :identifier, got :delimiter"},
{"dangling property access", "y = foo.\n", 2, 1, "Expected :identifier, got :eof"},
{"dangling method access", "y = foo:\n", 2, 1, "Expected :identifier, got :eof"},
{"table field with no key", "t = { = 1 }\n", 1, 7, "Expected expression"},
{"method call missing name", "a:()\n", 1, 3, "Expected :identifier, got :delimiter"},
{"if missing then", "if x y then end\n", 1, 6, "Expected :keyword::then"},
{"while missing do", "while c x end\n", 1, 9, "Expected :keyword::do"},
{"local missing name", "local = 1\n", 1, 7, "Expected identifier or 'function' after 'local'"}
]

@deep_propagation_errors [
# The highest-value cases: a bad token deep inside nested functions/calls
# must point at the innermost offending token, never the outer call line
# or the `function` keyword.
{"deep error in 5th call argument", ~s|w:step("a", "b", "c", {x = 1}, function()\n bar(\nend)\n|, 3, 1,
"Expected expression"},
{"deep error in a table field value", "t = {\n a = 1,\n b = foo(,\n}\n", 3, 11, "Expected expression"},
{"bad token in function inside function inside call",
"outer(function()\n inner(function()\n x = +\n end)\nend)\n", 3, 9, "Expected expression"},
{"deep error in a return list", "function f()\n return 1, 2, g(\nend\n", 3, 1, "Expected expression"},
{"stray statement after nested calls", "outer(mid(inner(\n)))\n.. bad\n", 3, 1, "bare"}
]

@statement_errors [
{"bare arithmetic expression", "2 + 2\n", 1, 3, "bare arithmetic"},
{"stray end keyword", "x = 1\nend\n", 2, 1, "Expected end of input"},
{"stray closing paren", "x = 1\n)\n", 2, 1, "Expected expression"},
{"double assignment operator", "a = = b\n", 1, 5, "Expected expression"},
{"invalid assignment target", "1 = 2\n", 1, 1, "syntax error near '='"}
]

describe "opener positions (unclosed brackets)" do
for {name, source, line, column, substring} <- @opener_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

describe "differential vs. reference Lua (luac -p)" do
@describetag :differential

setup do
case System.find_executable("luac") do
nil -> {:skip, "luac not found on PATH"}
path -> {:ok, luac: path}
describe "lexer-origin positions" do
for {name, source, line, column, substring} <- @lexer_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

for {name, source, _expected_line, _substring, :match} <- @corpus do
test "agrees with luac on: #{name}", %{luac: luac} do
assert {:error, msg} = Parser.parse(unquote(source))
ours = error_line(strip_ansi(msg))
describe "missing-end positions" do
for {name, source, line, column, substring} <- @missing_end_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

assert is_integer(ours)
assert reference_line(luac, unquote(source)) == ours
describe "concrete offending-token positions" do
for {name, source, line, column, substring} <- @offending_token_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

defp strip_ansi(string), do: String.replace(string, ~r/\e\[[0-9;]*m/, "")
describe "deep-propagation positions" do
for {name, source, line, column, substring} <- @deep_propagation_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

defp error_line(clean) do
case Regex.run(~r/at line (\d+)/, clean) do
[_, n] -> String.to_integer(n)
_ -> nil
describe "statement-level positions" do
for {name, source, line, column, substring} <- @statement_errors do
test name do
assert_error_at(unquote(source), unquote(line), unquote(column), unquote(substring))
end
end
end

defp reference_line(luac, source) do
path = Path.join(System.tmp_dir!(), "lua_error_position_#{:erlang.unique_integer([:positive])}.lua")
File.write!(path, source)
# Asserts the structured position (exact line + column) and the rendered
# banner + message together, so a drift in either the position or the
# human-facing string is caught.
defp assert_error_at(source, line, column, substring) do
assert {:error, [%Error{position: position} = error]} = Parser.parse_structured(source),
"expected exactly one structured error for:\n#{source}"

try do
{out, _status} = System.cmd(luac, ["-p", path], stderr_to_stdout: true)
assert position != nil, "expected a positioned error, got: #{inspect(error)}"

case Regex.run(~r/:(\d+):/, out) do
[_, n] -> String.to_integer(n)
_ -> nil
end
after
File.rm(path)
end
assert {position.line, position.column} == {line, column},
"expected position {#{line}, #{column}}, got {#{position.line}, #{position.column}} for:\n#{source}"

assert {:error, msg} = Parser.parse(source)
clean = strip_ansi(msg)

assert clean =~ "at line #{line}, column #{column}",
"expected rendered banner 'at line #{line}, column #{column}', got:\n#{clean}"

assert clean =~ substring,
"expected message to contain #{inspect(substring)}, got:\n#{clean}"
end

defp strip_ansi(string), do: String.replace(string, ~r/\e\[[0-9;]*m/, "")
end
Loading