Skip to content

Fix lexing MacroVar syntax in literal #16779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
straight-shoota wants to merge 3 commits into
base: master
Choose a base branch
from
+53 −108
Open

Fix lexing MacroVar syntax in literal #16779

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19af870
Add `#delimiter_state_for_precent_literal`
straight-shoota Mar 25, 2026
f5c5b63
Use `#delimiter_state_for_precent_literal` in precondition
straight-shoota Mar 25, 2026
04ece9f
Fix lexing `MacroVar` syntax in literal
straight-shoota Mar 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions spec/compiler/parser/parser_spec.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,9 @@ module Crystal
it_parses %(macro foo\n%q(%t{)\nend), Macro.new("foo", body: Expressions.from(["%q(%t".macro_literal, "{)\n".macro_literal] of ASTNode))
it_parses %(macro foo\n%(%t{)\nend), Macro.new("foo", body: Expressions.from(["%(%t".macro_literal, "{)\n".macro_literal] of ASTNode))

assert_syntax_error %({% begin %}\n%q(%t{})\n{% end %}), %(unexpected token: "}") # #16772
it_parses %({% begin %}\n%q(%t{})\n{% end %}), MacroIf.new(true.bool, Expressions.from(["\n%q(%t".macro_literal, "{})\n".macro_literal] of ASTNode))
it_parses %({% begin %}\n%(%t{})\n{% end %}), MacroIf.new(true.bool, Expressions.from(["\n%(%t".macro_literal, "{})\n".macro_literal] of ASTNode))
assert_syntax_error %({% begin %}\n%q(%t{)\n{% end %}), %{unexpected token: ")"}
it_parses %({% begin %}\n%q(%t{)\n{% end %}), MacroIf.new(true.bool, Expressions.from(["\n%q(%t".macro_literal, "{)\n".macro_literal] of ASTNode))
it_parses %({% begin %}\n%(%t{)\n{% end %}), MacroIf.new(true.bool, Expressions.from(["\n%(%t".macro_literal, "{)\n".macro_literal] of ASTNode))

it_parses ":foo", "foo".symbol
Expand Down Expand Up @@ -2244,6 +2244,8 @@ module Crystal
it_parses "{% begin %}%r#{open}\\A#{close}{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r#{open}\\A#{close}"))
end

it_parses "{% begin %}%-{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%-"))

it_parses %(foo(bar:"a", baz:"b")), Call.new("foo", named_args: [NamedArgument.new("bar", "a".string), NamedArgument.new("baz", "b".string)])
it_parses %(foo(bar:a, baz:b)), Call.new("foo", named_args: [NamedArgument.new("bar", "a".call), NamedArgument.new("baz", "b".call)])
it_parses %({foo:"a", bar:"b"}), NamedTupleLiteral.new([NamedTupleLiteral::Entry.new("foo", "a".string), NamedTupleLiteral::Entry.new("bar", "b".string)])
Expand Down
155 changes: 49 additions & 106 deletions src/compiler/crystal/syntax/lexer.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,67 +304,25 @@ module Crystal
if @wants_def_or_macro_name
next_char :OP_PERCENT
else
case next_char
when '='
next_char :OP_PERCENT_EQ
when '(', '[', '{', '<', '|'
delimited_pair :string, current_char, closing_char, start
when 'i'
case peek_next_char
when '(', '{', '[', '<', '|'
start_char = next_char
next_char :SYMBOL_ARRAY_START
@token.raw = "%i#{start_char}" if @wants_raw
@token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char))
else
@token.type = :OP_PERCENT
end
when 'q'
case peek_next_char
when '(', '{', '[', '<', '|'
next_char
delimited_pair :string, current_char, closing_char, start, allow_escapes: false
else
@token.type = :OP_PERCENT
end
when 'Q'
case peek_next_char
when '(', '{', '[', '<', '|'
next_char
delimited_pair :string, current_char, closing_char, start
else
@token.type = :OP_PERCENT
end
when 'r'
case peek_next_char
when '(', '[', '{', '<', '|'
next_char
delimited_pair :regex, current_char, closing_char, start
else
@token.type = :OP_PERCENT
end
when 'x'
case peek_next_char
when '(', '[', '{', '<', '|'
next_char
delimited_pair :command, current_char, closing_char, start
else
@token.type = :OP_PERCENT
end
when 'w'
case peek_next_char
when '(', '{', '[', '<', '|'
start_char = next_char
next_char :STRING_ARRAY_START
@token.raw = "%w#{start_char}" if @wants_raw
@token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char))
if delimiter_state = lookahead { delimiter_state_for_percent_literal }
@token.delimiter_state = delimiter_state
@token.type = case delimiter_state.kind
when .symbol_array? then Token::Kind::SYMBOL_ARRAY_START
when .string_array? then Token::Kind::STRING_ARRAY_START
else Token::Kind::DELIMITER_START
end

next_char
set_token_raw_from_start(start)
else
case next_char
when '='
next_char :OP_PERCENT_EQ
when '}'
next_char :OP_PERCENT_RCURLY
else
@token.type = :OP_PERCENT
end
when '}'
next_char :OP_PERCENT_RCURLY
else
@token.type = :OP_PERCENT
end
end
when '(' then next_char :OP_LPAREN
Expand Down Expand Up @@ -1784,43 +1742,17 @@ module Crystal
return @token
end

if !delimiter_state && current_char == '%' && ident_start?(peek_next_char)
if !delimiter_state && current_char == '%' && ident_start?(peek_next_char) && !peek_ahead { delimiter_state_for_percent_literal }
char = next_char
if char == 'q' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:string, current_char, closing_char)
next_char
elsif char == 'Q' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:string, current_char, closing_char)
next_char
elsif char == 'i' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:symbol_array, current_char, closing_char)
next_char
elsif char == 'w' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:string_array, current_char, closing_char)
next_char
elsif char == 'x' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:command, current_char, closing_char)
next_char
elsif char == 'r' && peek_next_char.in?('(', '<', '[', '{', '|')
next_char
delimiter_state = Token::DelimiterState.new(:regex, current_char, closing_char)
next_char
else
start = current_pos
while ident_part?(char)
char = next_char
end
beginning_of_line = false
@token.type = :MACRO_VAR
@token.value = string_range_from_pool(start)
@token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
return @token
start = current_pos
while ident_part?(char)
char = next_char
end
beginning_of_line = false
@token.type = :MACRO_VAR
@token.value = string_range_from_pool(start)
@token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
return @token
end

if !delimiter_state && current_char == 'e'
Expand Down Expand Up @@ -1898,19 +1830,13 @@ module Crystal
end
whitespace = false
when '%'
case char = peek_next_char
when '(', '[', '<', '{', '|'
next_char
delimiter_state = Token::DelimiterState.new(:string, char, closing_char)
else
whitespace = false
# Don't break if this looks like a prefixed percent literal that will
# be handled before the main loop (e.g., %Q|...|, %w|...|, etc.)
if !delimiter_state && ident_start?(char)
is_percent_literal =
char.in?('q', 'Q', 'w', 'i', 'r', 'x') &&
lookahead { next_char; peek_next_char.in?('(', '<', '[', '{', '|') }
break unless is_percent_literal
whitespace = false

if !delimiter_state
delimiter_state = lookahead { delimiter_state_for_percent_literal }
if !delimiter_state && ident_start?(peek_next_char)
# Start of a macro var
break
end
end
when '#'
Expand Down Expand Up @@ -2004,6 +1930,23 @@ module Crystal
@token
end

def delimiter_state_for_percent_literal
char = next_char
delimiter_kind = case char
when 'i' then Token::DelimiterKind::SYMBOL_ARRAY
when 'r' then Token::DelimiterKind::REGEX
when 'w' then Token::DelimiterKind::STRING_ARRAY
when 'x' then Token::DelimiterKind::COMMAND
when 'q', 'Q' then Token::DelimiterKind::STRING
when '(', '[', '{', '<', '|' then Token::DelimiterKind::STRING
else return
end

return unless char.in?('(', '<', '[', '{', '|') || next_char.in?('(', '<', '[', '{', '|')

Token::DelimiterState.new(delimiter_kind, current_char, closing_char, allow_escapes: !char.in?('q', 'w'))
end

def lookahead(preserve_token_on_fail = false, &)
old_pos, old_line, old_column = current_pos, @line_number, @column_number
@temp_token.copy_from(@token) if preserve_token_on_fail
Expand Down
Loading