Standardize parsing of symbol and string array literals

spec/compiler/formatter/formatter_spec.cr

@@ -1161,6 +1161,9 @@ describe Crystal::Formatter do
   assert_format "%w{one(   two(  three)}", "%w{one( two( three)}"
   assert_format "%i{one(   two(  three)}", "%i{one( two( three)}"
 
+  assert_format "%w(\n\n)\n# ```\n# 1\n# ```\n", "%w()\n# ```\n# 1\n# ```"
+  assert_format "%w(a\\ b)"
+
   assert_format "/foo/"
   assert_format "/foo/imx"
   assert_format "/foo \#{ bar }/", "/foo \#{bar}/"
@@ -1815,6 +1818,7 @@ describe Crystal::Formatter do
   assert_format "1 #=> 2", "1 # => 2"
   assert_format "1 #=>2", "1 # => 2"
   assert_format "foo(\n  [\n    1,\n    2,\n  ],\n  [\n    3,\n    4,\n  ]\n)"
+  assert_format "begin\n  %w(\n    one two\n    three four\n  )\nend"
   assert_format "%w(\n  one two\n  three four\n)"
   assert_format "a = %w(\n  one two\n  three four\n)"
   assert_format "foo &.bar do\n  1 + 2\nend"

spec/compiler/lexer/lexer_string_array_spec.cr

@@ -8,19 +8,23 @@ private def it_should_be_valid_string_array_lexer(lexer)
   token = lexer.next_token
   token.type.should eq(t :STRING_ARRAY_START)
 
-  token = lexer.next_string_array_token
+  token = lexer.next_string_token(token.delimiter_state)
   token.type.should eq(t :STRING)
   token.value.should eq("one")
 
-  token = lexer.next_string_array_token
+  token = lexer.next_string_token(token.delimiter_state)
+  token.type.should eq(t :SPACE)
+  token.value.should eq(" ")
+
+  token = lexer.next_string_token(token.delimiter_state)
   token.type.should eq(t :STRING)
   token.value.should eq("two")
 
-  token = lexer.next_string_array_token
+  token = lexer.next_string_token(token.delimiter_state)
   token.type.should eq(t :STRING_ARRAY_END)
 end
 
-describe "Lexer string array" do
+describe "Lexer %w string array" do
   it "lexes simple string array" do
     lexer = Lexer.new("%w(one two)")
 
@@ -33,25 +37,29 @@ describe "Lexer string array" do
     token = lexer.next_token
     token.type.should eq(t :STRING_ARRAY_START)
 
-    token = lexer.next_string_array_token
+    token = lexer.next_string_token(token.delimiter_state)
     token.type.should eq(t :STRING)
     token.value.should eq("one")
 
-    token = lexer.next_string_array_token
+    token = lexer.next_string_token(token.delimiter_state)
+    token.type.should eq(t :SPACE)
+    token.value.should eq(" \n ")
+
+    token = lexer.next_string_token(token.delimiter_state)
     token.type.should eq(t :STRING)
     token.value.should eq("two")
 
-    token = lexer.next_string_array_token
+    token = lexer.next_string_token(token.delimiter_state)
     token.type.should eq(t :STRING_ARRAY_END)
   end
 
   it "lexes string array with new line gives correct column for next token" do
     lexer = Lexer.new("%w(one \n two).")
 
-    lexer.next_token
-    lexer.next_string_array_token
-    lexer.next_string_array_token
-    lexer.next_string_array_token
+    token = lexer.next_token
+    lexer.next_string_token(token.delimiter_state)
+    lexer.next_string_token(token.delimiter_state)
+    lexer.next_string_token(token.delimiter_state)
 
     token = lexer.next_token
     token.line_number.should eq(2)

spec/compiler/parser/parser_spec.cr

@@ -3032,8 +3032,8 @@ module Crystal
         "/"   => regex("a\\\\ b"),
         "%x[" => command("a\\ b"),
         "`"   => command("a\\ b"),
-        "%w[" => string_array("a\\ b".string),
-        "%i[" => symbol_array("a\\ b".symbol),
+        "%w[" => string_array("a\\".string, "b".string),
+        "%i[" => symbol_array("a\\".symbol, "b".symbol),
         ":\"" => "a\\ b".symbol,
       }
       it_parses_literal "\\\\a", {
@@ -3045,8 +3045,8 @@ module Crystal
         "/"   => regex("\\\\a"),
         "%x[" => command("\\a"),
         "`"   => command("\\a"),
-        "%w[" => string_array("\\\\a".string),
-        "%i[" => symbol_array("\\\\a".symbol),
+        "%w[" => string_array("\\a".string),
+        "%i[" => symbol_array("\\a".symbol),
         ":\"" => "\\a".symbol,
       }
       it_parses_literal "\\", {
@@ -3071,8 +3071,8 @@ module Crystal
         "/"   => regex("\\\\"),
         "%x[" => command("\\"),
         "`"   => command("\\"),
-        "%w[" => "Unterminated string array literal", # FIXME: #12277
-        "%i[" => "Unterminated symbol array literal", # FIXME: #12277
+        "%w[" => string_array("\\".string),
+        "%i[" => symbol_array("\\".symbol),
         ":\"" => "\\".symbol,
       }
       it_parses_literal "\\\\\\", {

src/compiler/crystal/syntax/lexer.cr

@@ -315,7 +315,7 @@ module Crystal
               start_char = next_char
               next_char :SYMBOL_ARRAY_START
               @token.raw = "%i#{start_char}" if @wants_raw
-              @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char))
+              @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char), allow_escapes: false)
             else
               @token.type = :OP_PERCENT
             end
@@ -357,7 +357,7 @@ module Crystal
               start_char = next_char
               next_char :STRING_ARRAY_START
               @token.raw = "%w#{start_char}" if @wants_raw
-              @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char))
+              @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char), allow_escapes: false)
             else
               @token.type = :OP_PERCENT
             end
@@ -1444,13 +1444,13 @@ module Crystal
         end
       end
 
-      case current_char
+      case char = current_char
       when '\0'
         raise_unterminated_quoted delimiter_state
       when string_end
         next_char
         if string_open_count == 0
-          @token.type = :DELIMITER_END
+          @token.type = delimiter_state.kind.array? ? Token::Kind::STRING_ARRAY_END : Token::Kind::DELIMITER_END
         else
           @token.type = :STRING
           @token.value = string_end.to_s
@@ -1545,6 +1545,16 @@ module Crystal
               next_char
             end
           end
+        elsif delimiter_state.kind.array?
+          case char = next_char
+          when '\\', .ascii_whitespace?, string_end, string_nest
+            string_token_escape_value char.to_s
+          else
+            next_char
+            @token.type = :STRING
+            @token.value = string_range(start)
+            @token.invalid_escape = true
+          end
         else
           @token.type = :STRING
           @token.value = current_char.to_s
@@ -1570,17 +1580,34 @@ module Crystal
         @token.line_number = @line_number
         @token.column_number = @column_number
 
-        if delimiter_state.kind.heredoc?
+        case delimiter_state.kind
+        when .heredoc?
           unless check_heredoc_end delimiter_state
             next_string_token_noescape delimiter_state
             @token.value = string_range(start)
           end
+        when .array?
+          @token.type = :SPACE
         else
           @token.type = :STRING
           @token.value = is_slash_r ? "\r\n" : "\n"
         end
       else
-        next_string_token_noescape delimiter_state
+        if delimiter_state.kind.array?
+          if char.ascii_whitespace?
+            while char.ascii_whitespace?
+              handle_slash_r_slash_n_or_slash_n
+              incr_line_number if char == '\n'
+              char = next_char
+            end
+
+            @token.type = :SPACE
+          else
+            next_string_array_token_noescape delimiter_state
+          end
+        else
+          next_string_token_noescape delimiter_state
+        end
         @token.value = string_range(start)
       end
 
@@ -1645,7 +1672,9 @@ module Crystal
             when .regex?   then "Unterminated regular expression"
             when .heredoc?
               "Unterminated heredoc: can't find \"#{delimiter_state.end}\" anywhere before the end of file"
-            when .string? then "Unterminated string literal"
+            when .string?       then "Unterminated string literal"
+            when .string_array? then "Unterminated string array literal"
+            when .symbol_array? then "Unterminated symbol array literal"
             else
               ::raise "unreachable"
             end
@@ -1800,7 +1829,7 @@ module Crystal
           next_char
         elsif char == 'w' && peek_next_char.in?('(', '<', '[', '{', '|')
           next_char
-          delimiter_state = Token::DelimiterState.percent_literal(:string_array, current_char, closing_char)
+          delimiter_state = Token::DelimiterState.percent_literal(:string_array, current_char, closing_char, allow_escapes: false)
           next_char
         elsif char == 'x' && peek_next_char.in?('(', '<', '[', '{', '|')
           next_char
@@ -2307,77 +2336,15 @@ module Crystal
       set_token_raw_from_start(start)
     end
 
-    def next_string_array_token
-      while true
-        if current_char == '\n'
-          next_char
-          incr_line_number 1
-        elsif current_char.ascii_whitespace?
-          next_char
-        else
-          break
-        end
-      end
-
-      reset_token
-
-      if current_char == @token.delimiter_state.end
-        @token.raw = current_char.to_s if @wants_raw
-        next_char :STRING_ARRAY_END
-        return @token
-      end
-
-      start = current_pos
-      sub_start = start
-      value = String::Builder.new
-
-      escaped = false
-      while true
-        case current_char
-        when Char::ZERO
-          break # raise is handled by parser
-        when @token.delimiter_state.end
-          unless escaped
-            # For symmetric delimiters (like ||), don't use nesting logic
-            if @token.delimiter_state.nest == @token.delimiter_state.end || @token.delimiter_state.open_count == 0
-              break
-            else
-              @token.delimiter_state = @token.delimiter_state.with_open_count_delta(-1)
-            end
-          end
-        when @token.delimiter_state.nest
-          unless @token.delimiter_state.nest == @token.delimiter_state.end || escaped
-            @token.delimiter_state = @token.delimiter_state.with_open_count_delta(+1)
-          end
-        when .ascii_whitespace?
-          break unless escaped
-        else
-          if escaped
-            value << '\\'
-          end
-        end
-
-        escaped = current_char == '\\'
-        if escaped
-          value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
-          sub_start = current_pos + 1
-        end
+    def next_string_array_token_noescape(delimiter_state)
+      string_end = delimiter_state.end
+      string_nest = delimiter_state.nest
 
+      while !current_char.in?(string_end, string_nest, '\0', '\\', '#') && !current_char.ascii_whitespace?
         next_char
       end
 
-      if start == current_pos
-        @token.type = :EOF
-        return @token
-      end
-
-      value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
-
       @token.type = :STRING
-      @token.value = value.to_s
-      set_token_raw_from_start(start)
-
-      @token
     end
 
     def consume_loc_pragma

src/compiler/crystal/syntax/parser.cr

@@ -2406,32 +2406,62 @@ module Crystal
     end
 
     def parse_string_array
-      parse_string_or_symbol_array StringLiteral, "String"
+      strings, end_location = parse_string_or_symbol_array_strings do |pieces|
+        combine_pieces(pieces, @token.delimiter_state)
+      end
+
+      ArrayLiteral.new(strings, Path.global("String")).at_end(end_location)
     end
 
     def parse_symbol_array
-      parse_string_or_symbol_array SymbolLiteral, "Symbol"
+      strings, end_location = parse_string_or_symbol_array_strings do |pieces|
+        string = combine_stringliteral_pieces(pieces, @token.delimiter_state)
+        SymbolLiteral.new(string)
+      end
+
+      ArrayLiteral.new(strings, Path.global("Symbol")).at_end(end_location)
     end
 
-    def parse_string_or_symbol_array(klass, elements_type)
+    def parse_string_or_symbol_array_strings(&)
       strings = [] of ASTNode
+
+      while !@token.type.string_array_end?
+        if element = parse_percent_array_element { |pieces| yield pieces }
+          strings << element
+        end
+      end
+
+      check :STRING_ARRAY_END
+
+      end_location = @token.location
+
+      next_token
+
+      {strings, end_location}
+    end
+
+    def parse_percent_array_element(&)
+      pieces = [] of Piece
+      start_location = nil
       end_location = nil
+      delimiter_state = @token.delimiter_state
 
       while true
-        next_string_array_token
+        end_location = token_end_location
+        next_string_token(delimiter_state)
+        start_location ||= @token.location
+        delimiter_state = @token.delimiter_state
         case @token.type
         when .string?
-          strings << klass.new(@token.value.to_s).at(@token.location).at_end(token_end_location)
-        when .string_array_end?
-          end_location = token_end_location
-          next_token
-          break
+          pieces << Piece.new(@token.value.to_s, @token.line_number)
         else
-          raise "Unterminated #{elements_type.downcase} array literal"
+          break
         end
       end
 
-      ArrayLiteral.new(strings, Path.global(elements_type)).at_end(end_location)
+      return if pieces.empty?
+
+      (yield pieces).at(start_location).at_end(end_location)
     end
 
     def parse_empty_array_literal