diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index 9cb5ece89473..65b1f88b1050 100644 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -1,8 +1,11 @@ require "../../support/syntax" -private def it_lexes(string, type) +private def it_lexes(string, type, *, slash_is_regex : Bool? = nil) it "lexes #{string.inspect}" do lexer = Lexer.new string + unless (v = slash_is_regex).nil? + lexer.slash_is_regex = v + end token = lexer.next_token token.type.should eq(type) end @@ -99,7 +102,7 @@ end private def it_lexes_operators(ops) ops.each do |op| - it_lexes op.to_s, op + it_lexes op.to_s, op, slash_is_regex: false end end @@ -244,9 +247,9 @@ describe "Lexer" do it_lexes_char "'\\\\'", '\\' assert_syntax_error "'", "unterminated char literal" assert_syntax_error "'\\", "unterminated char literal" - it_lexes_operators [:"=", :"<", :"<=", :">", :">=", :"+", :"-", :"*", :"(", :")", + it_lexes_operators [:"=", :"<", :"<=", :">", :">=", :"+", :"-", :"*", :"/", :"//", :"(", :")", :"==", :"!=", :"=~", :"!", :",", :".", :"..", :"...", :"&&", :"||", - :"|", :"{", :"}", :"?", :":", :"+=", :"-=", :"*=", :"%=", :"&=", + :"|", :"{", :"}", :"?", :":", :"+=", :"-=", :"*=", :"/=", :"%=", :"//=", :"&=", :"|=", :"^=", :"**=", :"<<", :">>", :"%", :"&", :"|", :"^", :"**", :"<<=", :">>=", :"~", :"[]", :"[]=", :"[", :"]", :"::", :"<=>", :"=>", :"||=", :"&&=", :"===", :";", :"->", :"[]?", :"{%", :"{{", :"%}", :"@[", :"!~", @@ -260,7 +263,7 @@ describe "Lexer" do it_lexes_instance_var "@foo" it_lexes_class_var "@@foo" it_lexes_globals ["$foo", "$FOO", "$_foo", "$foo123"] - it_lexes_symbols [":foo", ":foo!", ":foo?", ":foo=", ":\"foo\"", ":かたな", ":+", ":-", ":*", ":/", + it_lexes_symbols [":foo", ":foo!", ":foo?", ":foo=", ":\"foo\"", ":かたな", ":+", ":-", ":*", ":/", "://", ":==", ":<", ":<=", ":>", ":>=", ":!", ":!=", ":=~", ":!~", ":&", ":|", ":^", ":~", ":**", ":>>", ":<<", ":%", ":[]", ":[]?", ":[]=", ":<=>", ":===", ":&+", ":&-", ":&*", ":&**"] @@ -497,13 +500,6 @@ describe "Lexer" do token.value.should eq ("a") end - it "lexes /=" do - lexer = Lexer.new("/=") - lexer.slash_is_regex = false - token = lexer.next_token - token.type.should eq(:"/=") - end - it "lexes != after identifier (#4815)" do lexer = Lexer.new("some_method!=5") token = lexer.next_token diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 02fedc732739..fcebeac232c9 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -422,7 +422,7 @@ module Crystal it_parses "f.x = Foo.new", Call.new("f".call, "x=", [Call.new("Foo".path, "new")] of ASTNode) it_parses "f.x = - 1", Call.new("f".call, "x=", [Call.new(1.int32, "-")] of ASTNode) - ["+", "-", "*", "/", "%", "|", "&", "^", "**", "<<", ">>", "&+", "&-", "&*"].each do |op| + ["+", "-", "*", "/", "//", "%", "|", "&", "^", "**", "<<", ">>", "&+", "&-", "&*"].each do |op| it_parses "f.x #{op}= 2", OpAssign.new(Call.new("f".call, "x"), op, 2.int32) end @@ -433,10 +433,11 @@ module Crystal it_parses "def %(); end;", Def.new("%") it_parses "def /(); end;", Def.new("/") - ["<<", "<", "<=", "==", ">>", ">", ">=", "+", "-", "*", "/", "%", "|", "&", "^", "**", "===", "=~", "!~", "&+", "&-", "&*", "&**"].each do |op| + ["<<", "<", "<=", "==", ">>", ">", ">=", "+", "-", "*", "/", "//", "%", "|", "&", "^", "**", "===", "=~", "!~", "&+", "&-", "&*", "&**"].each do |op| it_parses "1 #{op} 2", Call.new(1.int32, op, 2.int32) it_parses "n #{op} 2", Call.new("n".call, op, 2.int32) it_parses "def #{op}(); end", Def.new(op) + it_parses "macro #{op};end", Macro.new(op, [] of Arg, Expressions.new) end ["bar", "+", "-", "*", "/", "<", "<=", "==", ">", ">=", "%", "|", "&", "^", "**", "===", "=~", "!~"].each do |name| @@ -445,7 +446,7 @@ module Crystal it_parses "foo.#{name}(1, 2)", Call.new("foo".call, name, 1.int32, 2.int32) end - ["+", "-", "*", "/", "%", "|", "&", "^", "**", "<<", ">>", "&+", "&-", "&*"].each do |op| + ["+", "-", "*", "/", "//", "%", "|", "&", "^", "**", "<<", ">>", "&+", "&-", "&*"].each do |op| it_parses "a = 1; a #{op}= 1", [Assign.new("a".var, 1.int32), OpAssign.new("a".var, op, 1.int32)] it_parses "a = 1; a #{op}=\n1", [Assign.new("a".var, 1.int32), OpAssign.new("a".var, op, 1.int32)] it_parses "a.b #{op}=\n1", OpAssign.new(Call.new("a".call, "b"), op, 1.int32) @@ -647,7 +648,7 @@ module Crystal assert_syntax_error "#{keyword} ? 1 : 2", "void value expression" assert_syntax_error "+#{keyword}", "void value expression" - ["<<", "<", "<=", "==", ">>", ">", ">=", "+", "-", "*", "/", "%", "|", + ["<<", "<", "<=", "==", ">>", ">", ">=", "+", "-", "*", "/", "//", "%", "|", "&", "^", "**", "===", "&+", "&-", "&*", "&**"].each do |op| assert_syntax_error "#{keyword} #{op} 1", "void value expression" end @@ -785,6 +786,9 @@ module Crystal it_parses "{% unless 1; 2; end %}", MacroExpression.new(If.new(1.int32, Nop.new, 2.int32), output: false) it_parses "{%\n1\n2\n3\n%}", MacroExpression.new(Expressions.new([1.int32, 2.int32, 3.int32] of ASTNode), output: false) + it_parses "{{ 1 // 2 }}", MacroExpression.new(Expressions.from([Call.new(1.int32, "//", 2.int32)] of ASTNode)) + it_parses "{{ //.options }}", MacroExpression.new(Expressions.from([Call.new(RegexLiteral.new(StringLiteral.new("")), "options")] of ASTNode)) + it_parses "[] of Int", ([] of ASTNode).array_of("Int".path) it_parses "[1, 2] of Int", ([1.int32, 2.int32] of ASTNode).array_of("Int".path) @@ -794,8 +798,6 @@ module Crystal it_parses "macro foo;end", Macro.new("foo", [] of Arg, Expressions.new) it_parses "macro [];end", Macro.new("[]", [] of Arg, Expressions.new) - it_parses "macro %();end", Macro.new("%", [] of Arg, Expressions.new) - it_parses "macro /();end", Macro.new("/", [] of Arg, Expressions.new) it_parses %(macro foo; 1 + 2; end), Macro.new("foo", [] of Arg, Expressions.from([" 1 + 2; ".macro_literal] of ASTNode)) it_parses %(macro foo(x); 1 + 2; end), Macro.new("foo", ([Arg.new("x")]), Expressions.from([" 1 + 2; ".macro_literal] of ASTNode)) it_parses %(macro foo(x)\n 1 + 2; end), Macro.new("foo", ([Arg.new("x")]), Expressions.from([" 1 + 2; ".macro_literal] of ASTNode)) diff --git a/spec/compiler/parser/to_s_spec.cr b/spec/compiler/parser/to_s_spec.cr index b16dbb7901e8..9015a6a4a0bc 100644 --- a/spec/compiler/parser/to_s_spec.cr +++ b/spec/compiler/parser/to_s_spec.cr @@ -37,7 +37,10 @@ describe "ASTNode#to_s" do expect_to_s "@foo.bar" expect_to_s %(:foo) expect_to_s %(:"{") + expect_to_s %(%r()) + expect_to_s %(%r()imx) expect_to_s %(/hello world/) + expect_to_s %(/hello world/imx) expect_to_s %(/\\s/) expect_to_s %(/\\?/) expect_to_s %(/\\(group\\)/) diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 915ab8da7381..42509ea706a6 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -37,6 +37,7 @@ module Crystal @count_whitespace = false @slash_is_regex = true @wants_raw = false + @wants_def_or_macro_name = false @string_pool = string_pool || StringPool.new # When lexing macro tokens, when we encounter `#{` inside @@ -287,8 +288,17 @@ module Crystal line = @line_number column = @column_number char = next_char - if !@slash_is_regex && char == '=' + if (@wants_def_or_macro_name || !@slash_is_regex) && char == '/' + case next_char + when '=' + next_char :"//=" + else + @token.type = :"//" + end + elsif !@slash_is_regex && char == '=' next_char :"/=" + elsif @wants_def_or_macro_name + @token.type = :"/" elsif @slash_is_regex @token.type = :DELIMITER_START @token.delimiter_state = Token::DelimiterState.new(:regex, '/', '/') @@ -303,65 +313,69 @@ module Crystal @token.type = :"/" end when '%' - case next_char - when '=' - next_char :"%=" - when '(', '[', '{', '<', '|' - delimited_pair :string, current_char, closing_char, start - when 'i' - case peek_next_char - when '(', '{', '[', '<', '|' - start_char = next_char - next_char :SYMBOL_ARRAY_START - @token.raw = "%i#{start_char}" if @wants_raw - @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char)) - else - @token.type = :"%" - end - when 'q' - case peek_next_char - when '(', '{', '[', '<', '|' - next_char - delimited_pair :string, current_char, closing_char, start, allow_escapes: false - else - @token.type = :"%" - end - when 'Q' - case peek_next_char - when '(', '{', '[', '<', '|' - next_char - delimited_pair :string, current_char, closing_char, start - else - @token.type = :"%" - end - when 'r' - case next_char - when '(', '[', '{', '<', '|' - delimited_pair :regex, current_char, closing_char, start - else - raise "unknown %r char" - end - when 'x' + if @wants_def_or_macro_name + next_char :"%" + else case next_char + when '=' + next_char :"%=" when '(', '[', '{', '<', '|' - delimited_pair :command, current_char, closing_char, start - else - raise "unknown %x char" - end - when 'w' - case peek_next_char - when '(', '{', '[', '<', '|' - start_char = next_char - next_char :STRING_ARRAY_START - @token.raw = "%w#{start_char}" if @wants_raw - @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char)) + delimited_pair :string, current_char, closing_char, start + when 'i' + case peek_next_char + when '(', '{', '[', '<', '|' + start_char = next_char + next_char :SYMBOL_ARRAY_START + @token.raw = "%i#{start_char}" if @wants_raw + @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char)) + else + @token.type = :"%" + end + when 'q' + case peek_next_char + when '(', '{', '[', '<', '|' + next_char + delimited_pair :string, current_char, closing_char, start, allow_escapes: false + else + @token.type = :"%" + end + when 'Q' + case peek_next_char + when '(', '{', '[', '<', '|' + next_char + delimited_pair :string, current_char, closing_char, start + else + @token.type = :"%" + end + when 'r' + case next_char + when '(', '[', '{', '<', '|' + delimited_pair :regex, current_char, closing_char, start + else + raise "unknown %r char" + end + when 'x' + case next_char + when '(', '[', '{', '<', '|' + delimited_pair :command, current_char, closing_char, start + else + raise "unknown %x char" + end + when 'w' + case peek_next_char + when '(', '{', '[', '<', '|' + start_char = next_char + next_char :STRING_ARRAY_START + @token.raw = "%w#{start_char}" if @wants_raw + @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char)) + else + @token.type = :"%" + end + when '}' + next_char :"%}" else @token.type = :"%" end - when '}' - next_char :"%}" - else - @token.type = :"%" end when '(' then next_char :"(" when ')' then next_char :")" @@ -412,7 +426,12 @@ module Crystal symbol "*" end when '/' - next_char_and_symbol "/" + case next_char + when '/' + next_char_and_symbol "//" + else + symbol "/" + end when '=' case next_char when '=' @@ -694,10 +713,14 @@ module Crystal set_token_raw_from_start(start) when '"', '`' delimiter = current_char - next_char - @token.type = :DELIMITER_START - @token.delimiter_state = Token::DelimiterState.new(delimiter == '`' ? :command : :string, delimiter, delimiter) - set_token_raw_from_start(start) + if delimiter == '`' && @wants_def_or_macro_name + next_char :"`" + else + next_char + @token.type = :DELIMITER_START + @token.delimiter_state = Token::DelimiterState.new(delimiter == '`' ? :command : :string, delimiter, delimiter) + set_token_raw_from_start(start) + end when '0' scan_zero_number(start) when '1', '2', '3', '4', '5', '6', '7', '8', '9' diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 3e997e27bdf5..db971143a783 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -380,7 +380,8 @@ module Crystal atomic.doc = doc atomic end - when :"+=", :"-=", :"*=", :"/=", :"%=", :"|=", :"&=", :"^=", :"**=", :"<<=", :">>=", :"||=", :"&&=", :"&+=", :"&-=", :"&*=" + when :"+=", :"-=", :"*=", :"/=", :"//=", :"%=", :"|=", :"&=", :"^=", :"**=", :"<<=", :">>=", + :"||=", :"&&=", :"&+=", :"&-=", :"&*=" unexpected_token unless allow_ops break unless can_be_assigned?(atomic) @@ -531,7 +532,7 @@ module Crystal end end - parse_operator :mul_or_div, :pow, "Call.new left, method, [right] of ASTNode, name_column_number: method_column_number", %(:"*", :"/", :"%", :"&*") + parse_operator :mul_or_div, :pow, "Call.new left, method, [right] of ASTNode, name_column_number: method_column_number", %(:"*", :"/", :"//", :"%", :"&*") parse_operator :pow, :prefix, "Call.new left, method, [right] of ASTNode, name_column_number: method_column_number", %(:"**", :"&**") def parse_prefix @@ -552,7 +553,7 @@ module Crystal end end - AtomicWithMethodCheck = [:IDENT, :CONST, :"+", :"-", :"*", :"/", :"%", :"|", :"&", :"^", :"**", :"<<", :"<", :"<=", :"==", :"!=", :"=~", :"!~", :">>", :">", :">=", :"<=>", :"===", :"[]", :"[]=", :"[]?", :"[", :"&+", :"&-", :"&*", :"&**"] + AtomicWithMethodCheck = [:IDENT, :CONST, :"+", :"-", :"*", :"/", :"//", :"%", :"|", :"&", :"^", :"**", :"<<", :"<", :"<=", :"==", :"!=", :"=~", :"!~", :">>", :">", :">=", :"<=>", :"===", :"[]", :"[]=", :"[]?", :"[", :"&+", :"&-", :"&*", :"&**"] def parse_atomic_with_method location = @token.location @@ -666,7 +667,7 @@ module Crystal atomic = Call.new(atomic, "#{name}=", [arg] of ASTNode, name_column_number: name_column_number).at(location) next - when :"+=", :"-=", :"*=", :"/=", :"%=", :"|=", :"&=", :"^=", :"**=", :"<<=", :">>=", :"||=", :"&&=", :"&+=", :"&-=", :"&*=" + when :"+=", :"-=", :"*=", :"/=", :"//=", :"%=", :"|=", :"&=", :"^=", :"**=", :"<<=", :">>=", :"||=", :"&&=", :"&+=", :"&-=", :"&*=" method = @token.type.to_s.byte_slice(0, @token.type.to_s.size - 1) next_token_skip_space_or_newline value = parse_op_assign @@ -2772,19 +2773,14 @@ module Crystal next_token - case current_char - when '%' - next_char - @token.type = :"%" - @token.column_number += 1 - when '/' - next_char - @token.type = :"/" - @token.column_number += 1 - else - skip_space_or_newline - check DefOrMacroCheck1 - end + # Force lexer return if possible a def or macro name + # cases like: def `, def /, def // + # that in regular statements states for delimiters + # here must be treated as method names. + @wants_def_or_macro_name = true + skip_space_or_newline + check DefOrMacroCheck1 + @wants_def_or_macro_name = false push_def @@ -2880,6 +2876,7 @@ module Crystal def parse_macro_body(start_line, start_column, macro_state = Token::MacroState.default) skip_whitespace = check_macro_skip_whitespace + slash_is_regex! pieces = [] of ASTNode @@ -2975,6 +2972,7 @@ module Crystal def parse_percent_macro_expression raise "can't nest macro expressions", @token if @in_macro_expression + slash_is_regex! location = @token.location macro_exp = parse_macro_expression check_macro_expression_end @@ -3194,8 +3192,9 @@ module Crystal exp end - DefOrMacroCheck1 = [:IDENT, :CONST, :"<<", :"<", :"<=", :"==", :"===", :"!=", :"=~", :"!~", :">>", :">", :">=", :"+", :"-", :"*", :"/", :"!", :"~", :"%", :"&", :"|", :"^", :"**", :"[]", :"[]=", :"<=>", :"[]?", :"&+", :"&-", :"&*", :"&**"] - DefOrMacroCheck2 = [:"<<", :"<", :"<=", :"==", :"===", :"!=", :"=~", :"!~", :">>", :">", :">=", :"+", :"-", :"*", :"/", :"!", :"~", :"%", :"&", :"|", :"^", :"**", :"[]", :"[]?", :"[]=", :"<=>", :"&+", :"&-", :"&*", :"&**"] + DefOrMacroCheck1 = [:IDENT, :CONST, :"`", + :"<<", :"<", :"<=", :"==", :"===", :"!=", :"=~", :"!~", :">>", :">", :">=", :"+", :"-", :"*", :"/", :"//", :"!", :"~", :"%", :"&", :"|", :"^", :"**", :"[]", :"[]?", :"[]=", :"<=>", :"&+", :"&-", :"&*", :"&**"] + DefOrMacroCheck2 = [:"<<", :"<", :"<=", :"==", :"===", :"!=", :"=~", :"!~", :">>", :">", :">=", :"+", :"-", :"*", :"/", :"//", :"!", :"~", :"%", :"&", :"|", :"^", :"**", :"[]", :"[]?", :"[]=", :"<=>", :"&+", :"&-", :"&*", :"&**"] def parse_def_helper(is_abstract = false) push_def @@ -3208,23 +3207,14 @@ module Crystal next_token - case current_char - when '%' - next_char - @token.type = :"%" - @token.column_number += 1 - when '/' - next_char - @token.type = :"/" - @token.column_number += 1 - when '`' - next_char - @token.type = :"`" - @token.column_number += 1 - else - skip_space_or_newline - check DefOrMacroCheck1 - end + # Force lexer return if possible a def or macro name + # cases like: def `, def /, def // + # that in regular statements states for delimiters + # here must be treated as method names. + @wants_def_or_macro_name = true + skip_space_or_newline + check DefOrMacroCheck1 + @wants_def_or_macro_name = false receiver = nil @yields = nil diff --git a/src/compiler/crystal/syntax/to_s.cr b/src/compiler/crystal/syntax/to_s.cr index e3567de932d4..eeedc61960eb 100644 --- a/src/compiler/crystal/syntax/to_s.cr +++ b/src/compiler/crystal/syntax/to_s.cr @@ -985,14 +985,20 @@ module Crystal end def visit(node : RegexLiteral) - @str << '/' - case exp = node.value - when StringLiteral - Regex.append_source exp.value, @str - when StringInterpolation - visit_interpolation(exp) { |s| Regex.append_source s, @str } + if (exp = node.value).is_a?(StringLiteral) && exp.value.empty? + # // is not always an empty regex, sometimes is an operator + # so it's safer to emit empty regex as %r() + @str << "%r()" + else + @str << '/' + case exp = node.value + when StringLiteral + Regex.append_source exp.value, @str + when StringInterpolation + visit_interpolation(exp) { |s| Regex.append_source s, @str } + end + @str << '/' end - @str << '/' @str << 'i' if node.options.includes? Regex::Options::IGNORE_CASE @str << 'm' if node.options.includes? Regex::Options::MULTILINE @str << 'x' if node.options.includes? Regex::Options::EXTENDED