diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index ae7f2a39c60b..5eb2f93f3182 100644 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -164,7 +164,7 @@ describe "Lexer" do ["+1.0f32", "+1.0"], ["-1.0f32", "-1.0"], ["-0.0f32", "-0.0"], ["1_234.567_890_f32", "1234.567890"]] it_lexes_f64 ["1.0", ["1.0hello", "1.0"], "+1.0", "-1.0", ["1_234.567_890", "1234.567890"]] it_lexes_f32 [["1e+23_f32", "1e+23"], ["1.2e+23_f32", "1.2e+23"]] - it_lexes_f64 ["1e23", "1e-23", "1e+23", "1.2e+23", ["1e23f64", "1e23"], ["1.2e+23_f64", "1.2e+23"]] + it_lexes_f64 ["1e23", "1e-23", "1e+23", "1.2e+23", ["1e23f64", "1e23"], ["1.2e+23_f64", "1.2e+23"], "0e40", "2e01", ["2_e2", "2e2"], "1E40"] it_lexes_number :i8, ["1i8", "1"] it_lexes_number :i8, ["1_i8", "1"] @@ -225,6 +225,7 @@ describe "Lexer" do it_lexes_number :u64, ["10000000000000000000_u64", "10000000000000000000"] it_lexes_i64 [["0x3fffffffffffffff", "4611686018427387903"]] + it_lexes_i64 [["-0x8000000000000000_i64", "-9223372036854775808"]] it_lexes_i64 ["-9223372036854775808", "9223372036854775807"] it_lexes_u64 [["0xffffffffffffffff", "18446744073709551615"]] @@ -295,6 +296,8 @@ describe "Lexer" do assert_syntax_error "118446744073709551616_u64", "118446744073709551616 doesn't fit in an UInt64" assert_syntax_error "18446744073709551616_u64", "18446744073709551616 doesn't fit in an UInt64" assert_syntax_error "-1_u64", "Invalid negative value -1 for UInt64" + assert_syntax_error "-0_u64", "Invalid negative value -0 for UInt64" + assert_syntax_error "-0u64", "Invalid negative value -0 for UInt64" assert_syntax_error "18446744073709551616_i32", "18446744073709551616 doesn't fit in an Int32" assert_syntax_error "9999999999999999999_i32", "9999999999999999999 doesn't fit in an Int32" @@ -311,13 +314,21 @@ describe "Lexer" do assert_syntax_error "18446744073709551616_u128", "18446744073709551616 doesn't fit in an UInt64. UInt128 literals that don't fit in an UInt64 are currently not supported" assert_syntax_error "-1_u128", "Invalid negative value -1 for UInt128" + assert_syntax_error "1__1", "consecutive underscores in numbers aren't allowed" + assert_syntax_error "-3_", "trailing '_' in number" + assert_syntax_error "0b_10", "unexpected '_' in number" + assert_syntax_error "10e_10", "unexpected '_' in number" + assert_syntax_error "1_.1", "unexpected '_' in number" + assert_syntax_error "-0e_12", "unexpected '_' in number" + + assert_syntax_error "0_12", "octal constants should be prefixed with 0o" assert_syntax_error "0123", "octal constants should be prefixed with 0o" assert_syntax_error "00", "octal constants should be prefixed with 0o" assert_syntax_error "01_i64", "octal constants should be prefixed with 0o" - assert_syntax_error "0xFF_i8", "255 doesn't fit in an Int8" - assert_syntax_error "0o200_i8", "128 doesn't fit in an Int8" - assert_syntax_error "0b10000000_i8", "128 doesn't fit in an Int8" + assert_syntax_error "0xFF_i8", "0xFF doesn't fit in an Int8" + assert_syntax_error "0o200_i8", "0o200 doesn't fit in an Int8" + assert_syntax_error "0b10000000_i8", "0b10000000 doesn't fit in an Int8" # 2**31 - 1 it_lexes_i32 [["0x7fffffff", "2147483647"], ["0o17777777777", "2147483647"], ["0b1111111111111111111111111111111", "2147483647"]] @@ -326,28 +337,30 @@ describe "Lexer" do it_lexes_i64 [["0xffffffff", "4294967295"], ["0o37777777777", "4294967295"], ["0b11111111111111111111111111111111", "4294967295"]] # 2**32 it_lexes_i64 [["0x100000000", "4294967296"], ["0o40000000000", "4294967296"], ["0b100000000000000000000000000000000", "4294967296"]] - assert_syntax_error "0x100000000i32", "4294967296 doesn't fit in an Int32" - assert_syntax_error "0o40000000000i32", "4294967296 doesn't fit in an Int32" - assert_syntax_error "0b100000000000000000000000000000000i32", "4294967296 doesn't fit in an Int32" + assert_syntax_error "0x100000000i32", "0x100000000 doesn't fit in an Int32" + assert_syntax_error "0o40000000000i32", "0o40000000000 doesn't fit in an Int32" + assert_syntax_error "0b100000000000000000000000000000000i32", "0b100000000000000000000000000000000 doesn't fit in an Int32" # 2**63 - 1 it_lexes_i64 [["0x7fffffffffffffff", "9223372036854775807"], ["0o777777777777777777777", "9223372036854775807"], ["0b111111111111111111111111111111111111111111111111111111111111111", "9223372036854775807"]] # 2**63 it_lexes_u64 [["0x8000000000000000", "9223372036854775808"], ["0o1000000000000000000000", "9223372036854775808"], ["0b1000000000000000000000000000000000000000000000000000000000000000", "9223372036854775808"]] - assert_syntax_error "0x8000000000000000i64", "9223372036854775808 doesn't fit in an Int64" - assert_syntax_error "0o1000000000000000000000i64", "9223372036854775808 doesn't fit in an Int64" - assert_syntax_error "0b1000000000000000000000000000000000000000000000000000000000000000i64", "9223372036854775808 doesn't fit in an Int64" + assert_syntax_error "0x8000000000000000i64", "0x8000000000000000 doesn't fit in an Int64" + assert_syntax_error "0o1000000000000000000000i64", "0o1000000000000000000000 doesn't fit in an Int64" + assert_syntax_error "0b1000000000000000000000000000000000000000000000000000000000000000i64", "0b1000000000000000000000000000000000000000000000000000000000000000 doesn't fit in an Int64" # 2**64 - 1 it_lexes_u64 [["0xffff_ffff_ffff_ffff", "18446744073709551615"], ["0o177777_77777777_77777777", "18446744073709551615"], ["0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111", "18446744073709551615"]] it_lexes_u64 [["0x00ffffffffffffffff", "18446744073709551615"], ["0o001777777777777777777777", "18446744073709551615"], ["0b001111111111111111111111111111111111111111111111111111111111111111", "18446744073709551615"]] # 2**64 assert_syntax_error "0x10000_0000_0000_0000", "0x10000_0000_0000_0000 doesn't fit in an UInt64" + assert_syntax_error "0x10000_0000_0000_0000_u64", "0x10000_0000_0000_0000 doesn't fit in an UInt64" + assert_syntax_error "0xfffffffffffffffff_u64", "0xfffffffffffffffff doesn't fit in an UInt64" assert_syntax_error "0o200000_00000000_00000000", "0o200000_00000000_00000000 doesn't fit in an UInt64" assert_syntax_error "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000", "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000 doesn't fit in an UInt64" # Very large assert_syntax_error "0x1afafafafafafafafafafaf", "0x1afafafafafafafafafafaf doesn't fit in an UInt64" - assert_syntax_error "0x1afafafafafafafafafafafi32", "0x1afafafafafafafafafafafi32 doesn't fit in an Int32" + assert_syntax_error "0x1afafafafafafafafafafafi32", "0x1afafafafafafafafafafaf doesn't fit in an Int32" assert_syntax_error "0o1234567123456712345671234567", "0o1234567123456712345671234567 doesn't fit in an UInt64" - assert_syntax_error "0o12345671234567_12345671234567_i8", "0o12345671234567_12345671234567_i8 doesn't fit in an Int8" + assert_syntax_error "0o12345671234567_12345671234567_i8", "0o12345671234567_12345671234567 doesn't fit in an Int8" assert_syntax_error "0b100000000000000000000000000000000000000000000000000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000000000 doesn't fit in an UInt64" it_lexes_i64 [["0o700000000000000000000", "8070450532247928832"]] @@ -356,14 +369,31 @@ describe "Lexer" do assert_syntax_error "4f33", "invalid float suffix" assert_syntax_error "4f65", "invalid float suffix" assert_syntax_error "4f22", "invalid float suffix" + assert_syntax_error "4i33", "invalid int suffix" + assert_syntax_error "4i65", "invalid int suffix" + assert_syntax_error "4i22", "invalid int suffix" + assert_syntax_error "4i3", "invalid int suffix" + assert_syntax_error "4i12", "invalid int suffix" + assert_syntax_error "4u33", "invalid uint suffix" + assert_syntax_error "4u65", "invalid uint suffix" + assert_syntax_error "4u22", "invalid uint suffix" + assert_syntax_error "4u3", "invalid uint suffix" + assert_syntax_error "4u12", "invalid uint suffix" # Tests for #8782 assert_syntax_error "4F32", %(unexpected token: "F32") assert_syntax_error "4F64", %(unexpected token: "F64") assert_syntax_error "0F32", %(unexpected token: "F32") + assert_syntax_error "4.0_u32", "Invalid suffix u32 for decimal number" + assert_syntax_error "2e8i8", "Invalid suffix i8 for decimal number" + assert_syntax_error ".42", ".1 style number literal is not supported, put 0 before dot" assert_syntax_error "-.42", ".1 style number literal is not supported, put 0 before dot" + assert_syntax_error "2e", "unexpected token: \"e\"" + assert_syntax_error "2ef32", "unexpected token: \"ef32\"" + assert_syntax_error "2e+_2", "unexpected '_' in number" + it "lexes not instance var" do lexer = Lexer.new "!@foo" token = lexer.next_token diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 2305cee8a689..1f422affc169 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -287,10 +287,8 @@ module Crystal case next_char when '=' next_char :"+=" - when '0' - scan_zero_number(start) - when '1', '2', '3', '4', '5', '6', '7', '8', '9' - scan_number(start) + when '0'..'9' + scan_number start when '+' raise "postfix increment is not supported, use `exp += 1`" else @@ -303,9 +301,7 @@ module Crystal next_char :"-=" when '>' next_char :"->" - when '0' - scan_zero_number start, negative: true - when '1', '2', '3', '4', '5', '6', '7', '8', '9' + when '0'..'9' scan_number start, negative: true when '-' raise "postfix decrement is not supported, use `exp -= 1`" @@ -777,10 +773,8 @@ module Crystal @token.delimiter_state = Token::DelimiterState.new(delimiter == '`' ? :command : :string, delimiter, delimiter) set_token_raw_from_start(start) end - when '0' - scan_zero_number(start) - when '1', '2', '3', '4', '5', '6', '7', '8', '9' - scan_number current_pos + when '0'..'9' + scan_number start when '@' start = current_pos case next_char @@ -1456,567 +1450,192 @@ module Crystal @token.raw = ":#{value}" if @wants_raw end - def scan_number(start, negative = false) - @token.type = :NUMBER - - has_underscore = false - is_integer = true - has_suffix = true - suffix_size = 0 - - while true - char = next_char - if char.ascii_number? - # Nothing to do - elsif char == '_' - has_underscore = true - else - break - end - end - - case current_char - when '.' - if peek_next_char.ascii_number? - is_integer = false - - while true - char = next_char - if char.ascii_number? - # Nothing to do - elsif char == '_' - has_underscore = true - else - break - end - end - - if current_char == 'e' || current_char == 'E' - next_char - - if current_char == '+' || current_char == '-' - next_char - end - - while true - if current_char.ascii_number? - # Nothing to do - elsif current_char == '_' - has_underscore = true - else - break - end - next_char - end - end - - if current_char == 'f' - suffix_size = consume_float_suffix - else - @token.number_kind = :f64 - end - else - @token.number_kind = :i32 - has_suffix = false - end - when 'e', 'E' - is_integer = false - next_char - - if current_char == '+' || current_char == '-' - next_char - end - - while true - if current_char.ascii_number? - # Nothing to do - elsif current_char == '_' - has_underscore = true - else - break - end - next_char - end - - if current_char == 'f' - suffix_size = consume_float_suffix - else - @token.number_kind = :f64 - end - when 'f' - is_integer = false - suffix_size = consume_float_suffix - when 'i' - suffix_size = consume_int_suffix - when 'u' - suffix_size = consume_uint_suffix - else - has_suffix = false - @token.number_kind = :i32 - end - - end_pos = current_pos - suffix_size - - if end_pos - start == 1 - # For numbers such as 0, 1, 2, 3, etc., we use a string from the poll - string_value = string_range_from_pool(start, end_pos) - else - string_value = string_range(start, end_pos) - end - string_value = string_value.delete('_') if has_underscore - - if is_integer - num_size = string_value.size - num_size -= 1 if negative - - if has_suffix - check_integer_literal_fits_in_size string_value, num_size, negative, start - else - deduce_integer_kind string_value, num_size, negative, start - end - end - - @token.value = string_value - set_token_raw_from_start(start) - end + macro gen_check_int_fits_in_size(type, method, size, number_size, raw_number_string, start, pos_before_suffix, negative, actual_type = nil) + {% if type.stringify.starts_with? "U" %} + raise "Invalid negative value #{string_range({{start}}, {{pos_before_suffix}})} for {{type}}", @token, (current_pos - {{start}}) if {{negative}} + {% end %} - macro gen_check_int_fits_in_size(type, method, size, *, actual_type = nil) - if num_size >= 20 + if !@token.value || {{number_size}} > {{size}} || ({{number_size}} == {{size}} && {{raw_number_string}}.to_{{method.id}}? == nil) {% if actual_type.nil? %} - raise_value_doesnt_fit_in "{{type}}", string_value, start + raise_value_doesnt_fit_in "{{type}}", {{start}}, {{pos_before_suffix}} {% else %} - raise_value_restricted_by "{{actual_type}}", "{{type}}", string_value, start + raise("#{string_range({{start}}, {{pos_before_suffix}})} doesn't fit in an {{actual_type}}. {{type}} literals that don't fit in an {{actual_type}} are currently not supported", @token, current_pos - {{start}}) {% end %} end - if num_size >= {{size}} - int_value = absolute_integer_value(string_value, negative) - max = {{type}}::MAX.{{method}} - max += 1 if negative - - if int_value > max - {% if actual_type.nil? %} - raise_value_doesnt_fit_in "{{type}}", string_value, start - {% else %} - raise_value_restricted_by "{{actual_type}}", "{{type}}", string_value, start - {% end %} - end - end end - macro gen_check_uint_fits_in_size(type, size) - if negative - raise "Invalid negative value #{string_value} for {{type}}" - end - if num_size >= 20 - raise_value_doesnt_fit_in "{{type}}", string_value, start - end - if num_size >= {{size}} - int_value = absolute_integer_value(string_value, negative) - if int_value > {{type}}::MAX - raise_value_doesnt_fit_in "{{type}}", string_value, start - end - end - end - - def check_integer_literal_fits_in_size(string_value, num_size, negative, start) - case @token.number_kind - when :i8 - gen_check_int_fits_in_size Int8, to_u8, 3 - when :u8 - gen_check_uint_fits_in_size UInt8, 3 - when :i16 - gen_check_int_fits_in_size Int16, to_u16, 5 - when :u16 - gen_check_uint_fits_in_size UInt16, 5 - when :i32 - gen_check_int_fits_in_size Int32, to_u32, 10 - when :u32 - gen_check_uint_fits_in_size UInt32, 10 - when :i64 - gen_check_int_fits_in_size Int64, to_u64, 19 - when :u64 - if negative - raise "Invalid negative value #{string_value} for UInt64" - end - - check_value_fits_in_uint64 string_value, num_size, start - when :i128 - gen_check_int_fits_in_size Int64, to_u64, 19, actual_type: Int128 - when :u128 - if negative - raise "Invalid negative value #{string_value} for UInt128" - end - - check_value_fits_in_uint64 string_value, num_size, start, actual_type: UInt128 - end + def raise_value_doesnt_fit_in(type, start, pos_before_suffix) + raise "#{string_range(start, pos_before_suffix)} doesn't fit in an #{type}", @token, (current_pos - start) end - def deduce_integer_kind(string_value, num_size, negative, start) - if negative - check_negative_value_fits_in_int64 string_value, num_size, start - else - check_value_fits_in_uint64 string_value, num_size, start - end - - if num_size >= 10 - int_value = absolute_integer_value(string_value, negative) - - int64max = Int64::MAX.to_u64 - int64max += 1 if negative - - int32max = Int32::MAX.to_u32 - int32max += 1 if negative - - if int_value > int64max - @token.number_kind = :u64 - elsif int_value > int32max - @token.number_kind = :i64 - end - end - end - - def absolute_integer_value(string_value, negative) - if negative - string_value[1..-1].to_u64 - else - string_value.to_u64 - end - end - - def check_negative_value_fits_in_int64(string_value, num_size, start) - if num_size > 19 - raise_value_doesnt_fit_in "Int64", string_value, start - end - - if num_size == 19 - i = 1 # skip '-' - "9223372036854775808".each_byte do |byte| - string_byte = string_value.byte_at(i) - if string_byte > byte - raise_value_doesnt_fit_in "Int64", string_value, start - elsif string_byte < byte - break - end - i += 1 - end - end - end - - def check_value_fits_in_uint64(string_value, num_size, start, actual_type = UInt64) - if num_size > 20 - if actual_type == UInt64 - raise_value_doesnt_fit_in "UInt64", string_value, start - else - raise_value_restricted_by actual_type, "UInt64", string_value, start + private def scan_number(start, negative = false) + @token.type = :NUMBER + base = 10 + number_size = 0 + suffix_size = 0 + is_decimal = false + is_e_notation = false + has_underscores = false + last_is_underscore = false + pos_after_prefix = start + + # Consume prefix + if current_char == '0' + case next_char + when 'b' then base = 2 + when 'o' then base = 8 + when 'x' then base = 16 + when '0'..'9' then raise("octal constants should be prefixed with 0o", @token, (current_pos - start)) + when '_' + raise("octal constants should be prefixed with 0o", @token, (current_pos - start)) if next_char.in?('0'..'9') + has_underscores = last_is_underscore = true end - end - if num_size == 20 - i = 0 - "18446744073709551615".each_byte do |byte| - string_byte = string_value.byte_at(i) - if string_byte > byte - if actual_type == UInt64 - raise_value_doesnt_fit_in "UInt64", string_value, start - else - raise_value_restricted_by actual_type, "UInt64", string_value, start - end - elsif string_byte < byte - break - end - i += 1 + # Skip prefix (b, o, x) + unless base == 10 + next_char + pos_after_prefix = current_pos + # Enforce number after prefix (disallow ex. "0x", "0x_1") + raise("unexpected '_' in number", @token, (current_pos - start)) if current_char == '_' + raise("numeric literal without digits", @token, (current_pos - start)) unless String::CHAR_TO_DIGIT[current_char.ord].to_u8! < base end end - end - - def raise_value_doesnt_fit_in(type, string_value, start) - raise "#{string_value} doesn't fit in an #{type}", @token, (current_pos - start) - end - - def raise_value_restricted_by(type, restricted_by_type, string_value, start) - raise "#{string_value} doesn't fit in an #{restricted_by_type}. #{type} literals that don't fit in an #{restricted_by_type} are currently not supported", @token, (current_pos - start) - end - def scan_zero_number(start, negative = false) - case peek_next_char - when 'x' - scan_hex_number(start, negative) - when 'o' - scan_octal_number(start, negative) - when 'b' - scan_bin_number(start, negative) - when '.' - scan_number(start) - when 'i' - @token.type = :NUMBER - @token.value = "0" - next_char - consume_int_suffix - set_token_raw_from_start(start) - when 'f' - @token.type = :NUMBER - @token.value = "0" - next_char - consume_float_suffix - set_token_raw_from_start(start) - when 'u' - @token.type = :NUMBER - @token.value = "0" - next_char - consume_uint_suffix - set_token_raw_from_start(start) - when '_' - scan_number(start) - else - if next_char.ascii_number? - raise "octal constants should be prefixed with 0o" - else - finish_scan_prefixed_number 0_u64, false, start + # Consume number + loop do + while String::CHAR_TO_DIGIT[current_char.ord].to_u8! < base + number_size += 1 unless number_size == 0 && current_char == '0' + next_char + last_is_underscore = false end - end - end - - def scan_bin_number(start, negative) - next_char - num = 0_u64 - num_size = 0 - while true - case next_char - when '0' - num = num << 1 - if num_size > 0 - num_size += 1 - end - when '1' - num = (num << 1) + 1 - num_size += 1 + case current_char when '_' - # Nothing - else + raise("consecutive underscores in numbers aren't allowed", @token, (current_pos - start)) if last_is_underscore + has_underscores = last_is_underscore = true + when '.' + raise("unexpected '_' in number", @token, (current_pos - start)) if last_is_underscore + break if is_decimal || base != 10 || !peek_next_char.in?('0'..'9') + is_decimal = true + when 'e', 'E' + last_is_underscore = false + break if is_e_notation || base != 10 + is_e_notation = is_decimal = true + next_char if peek_next_char.in?('+', '-') + raise("unexpected '_' in number", @token, (current_pos - start)) if peek_next_char == '_' + break unless peek_next_char.in?('0'..'9') + when 'i', 'u', 'f' + before_suffix_pos = current_pos + @token.number_kind = consume_number_suffix + next_char + suffix_size = current_pos - before_suffix_pos + suffix_size += 1 if last_is_underscore break - end - end - - num = nil if num_size > 64 - finish_scan_prefixed_number num, negative, start - end - - def scan_octal_number(start, negative) - next_char - - num = 0_u64 - num_size = first_digit = 0 - while true - char = next_char - if '0' <= char <= '7' - num = (num << 3) | (char - '0') - if num_size == 0 - first_digit = num - num_size += 1 if char != '0' - else - num_size += 1 - end - elsif char == '_' else + raise("trailing '_' in number", @token, (current_pos - start)) if last_is_underscore break end - end - - # 0o177777_77777777_77777777 is the largest UInt64. - num = nil if {num_size, first_digit} > {22, 0o1} - finish_scan_prefixed_number num, negative, start - end - def scan_hex_number(start, negative = false) - next_char - - num = 0_u64 - num_size = 0 - while true - char = next_char - if char == '_' - else - hex_value = char_to_hex(char) { nil } - if hex_value - num = (num << 4) | hex_value - if num_size > 0 || char != '0' - num_size += 1 - end - else - break - end - end + next_char end - # 0xFFFF_FFFF_FFFF_FFFF is the longest UInt64. - num = nil if num_size > 16 - finish_scan_prefixed_number num, negative, start - end + set_token_raw_from_start(start) - def finish_scan_prefixed_number(num : Int?, negative : Bool, start : Int32) - if num.nil? # Doesn't even fit in UInt64 - case current_char - when 'i' - consume_int_suffix - when 'u' - consume_uint_suffix - else - @token.number_kind = :u64 - end - case @token.number_kind - when :i8, :i16, :i32, :i64, :i128 - type_name = "Int" + @token.number_kind.to_s[1..] - when :u8, :u16, :u32, :u64, :u128 - type_name = "UInt" + @token.number_kind.to_s[1..] - else - raise "BUG: Expecting an integer token, got #{@token.number_kind}" + # Sanitize string (or convert to decimal unless number is in base 10) + pos_before_suffix = current_pos - suffix_size + raw_number_string = string_range(pos_after_prefix, pos_before_suffix) + if base == 10 + raw_number_string = raw_number_string.delete('_') if has_underscores + @token.value = raw_number_string + else + base10_number_string = raw_number_string.to_u64?(base: base, underscore: true).try &.to_s + if base10_number_string + number_size = base10_number_string.size + first_byte = @reader.string.byte_at(start).chr + base10_number_string = first_byte + base10_number_string if first_byte.in?('+', '-') + @token.value = raw_number_string = base10_number_string end - raise_value_doesnt_fit_in type_name, string_range_from_pool(start), start end - if negative - string_value = (num.to_i64 * -1).to_s - else - string_value = num.to_s + if is_decimal + @token.number_kind = :f64 if suffix_size == 0 + raise("Invalid suffix #{@token.number_kind} for decimal number", @token, (current_pos - start)) unless @token.number_kind.in?(:f32, :f64) + return end - name_size = string_value.size - name_size -= 1 if negative - - case current_char - when 'i' - consume_int_suffix - check_integer_literal_fits_in_size string_value, name_size, negative, start - when 'u' - consume_uint_suffix - check_integer_literal_fits_in_size string_value, name_size, negative, start + # Check or determine suffix + if suffix_size == 0 + raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix) unless @token.value + @token.number_kind = case number_size + when 0..9 then :i32 + when 10 then raw_number_string.to_i32? ? :i32 : :i64 + when 11..18 then :i64 + when 19 + if raw_number_string.to_i64? + :i64 + elsif negative + raise_value_doesnt_fit_in(Int64, start, pos_before_suffix) + else + :u64 + end + when 20 + raise_value_doesnt_fit_in(Int64, start, pos_before_suffix) if negative + raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix) unless raw_number_string.to_u64? + :u64 + else + raise_value_doesnt_fit_in(Int64, start, pos_before_suffix) if negative + raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix) + end else - @token.number_kind = :i32 - deduce_integer_kind string_value, name_size, negative, start - end - - first_byte = @reader.string.byte_at(start) - if first_byte === '+' - string_value = "+#{string_value}" - elsif first_byte === '-' && num == 0 - string_value = "-0" + case @token.number_kind + when :i8 then gen_check_int_fits_in_size(Int8, :i8, 3, number_size, raw_number_string, start, pos_before_suffix, negative) + when :u8 then gen_check_int_fits_in_size(UInt8, :u8, 3, number_size, raw_number_string, start, pos_before_suffix, negative) + when :i16 then gen_check_int_fits_in_size(Int16, :i16, 5, number_size, raw_number_string, start, pos_before_suffix, negative) + when :u16 then gen_check_int_fits_in_size(UInt16, :u16, 5, number_size, raw_number_string, start, pos_before_suffix, negative) + when :i32 then gen_check_int_fits_in_size(Int32, :i32, 10, number_size, raw_number_string, start, pos_before_suffix, negative) + when :u32 then gen_check_int_fits_in_size(UInt32, :u32, 10, number_size, raw_number_string, start, pos_before_suffix, negative) + when :i64 then gen_check_int_fits_in_size(Int64, :i64, 19, number_size, raw_number_string, start, pos_before_suffix, negative) + when :u64 then gen_check_int_fits_in_size(UInt64, :u64, 20, number_size, raw_number_string, start, pos_before_suffix, negative) + when :i128 then gen_check_int_fits_in_size(Int128, :i64, 19, number_size, raw_number_string, start, pos_before_suffix, negative, Int64) + when :u128 then gen_check_int_fits_in_size(UInt128, :u64, 20, number_size, raw_number_string, start, pos_before_suffix, negative, UInt64) + end end - - @token.type = :NUMBER - @token.value = string_value - set_token_raw_from_start(start) end - def consume_int_suffix - case next_char - when '8' - next_char - @token.number_kind = :i8 - 2 - when '1' + private def consume_number_suffix : Symbol + case current_char + when 'i' case next_char - when '2' - if next_char == '8' - next_char - @token.number_kind = :i128 - 4 - else - raise "invalid int suffix" + when '8' then return :i8 + when '1' + case next_char + when '2' then return :i128 if next_char == '8' + when '6' then return :i16 end - when '6' - next_char - @token.number_kind = :i16 - 3 - else - raise "invalid int suffix" - end - when '3' - if next_char == '2' - next_char - @token.number_kind = :i32 - 3 - else - raise "invalid int suffix" - end - when '6' - if next_char == '4' - next_char - @token.number_kind = :i64 - 3 - else - raise "invalid int suffix" + when '3' then return :i32 if next_char == '2' + when '6' then return :i64 if next_char == '4' end - else raise "invalid int suffix" - end - end - - def consume_uint_suffix - case next_char - when '8' - next_char - @token.number_kind = :u8 - 2 - when '1' + when 'u' case next_char - when '2' - if next_char == '8' - next_char - @token.number_kind = :u128 - 4 - else - raise "invalid uint suffix" + when '8' then return :u8 + when '1' + case next_char + when '2' then return :u128 if next_char == '8' + when '6' then return :u16 end - when '6' - next_char - @token.number_kind = :u16 - 3 - else - raise "invalid uint suffix" - end - when '3' - if next_char == '2' - next_char - @token.number_kind = :u32 - 3 - else - raise "invalid uint suffix" + when '3' then return :u32 if next_char == '2' + when '6' then return :u64 if next_char == '4' end - when '6' - if next_char == '4' - next_char - @token.number_kind = :u64 - 3 - else - raise "invalid uint suffix" - end - else raise "invalid uint suffix" - end - end - - def consume_float_suffix - case next_char - when '3' - if next_char == '2' - next_char - @token.number_kind = :f32 - 3 - else - raise "invalid float suffix" - end - when '6' - if next_char == '4' - next_char - @token.number_kind = :f64 - 3 - else - raise "invalid float suffix" + when 'f' + case next_char + when '3' then return :f32 if next_char == '2' + when '6' then return :f64 if next_char == '4' end - else raise "invalid float suffix" end + raise "BUG: invalid suffix" end def next_string_token(delimiter_state)