Merge branch 'feature/fix-php-7.4-numeric-separators-backfill' of https://github.com/jrfnl/PHP_CodeSniffer

Author: gsherwood

src/Tokenizers/PHP.php

@@ -992,20 +992,24 @@ protected function tokenize($string)
 
                     if ($tokens[$i][0] === T_LNUMBER
                         || $tokens[$i][0] === T_DNUMBER
-                        || ($tokens[$i][0] === T_STRING
-                        && $tokens[$i][1][0] === '_')
                     ) {
                         $newContent .= $tokens[$i][1];
+                        continue;
+                    }
 
-                        // Any T_DNUMBER token needs to make the
-                        // new number a T_DNUMBER as well.
-                        if ($tokens[$i][0] === T_DNUMBER) {
-                            $newType = T_DNUMBER;
-                        }
+                    if ($tokens[$i][0] === T_STRING
+                        && $tokens[$i][1][0] === '_'
+                        && ((strpos($newContent, '0x') === 0
+                        && preg_match('`^((?<!\.)_[0-9A-F][0-9A-F\.]*)+$`iD', $tokens[$i][1]) === 1)
+                        || (strpos($newContent, '0x') !== 0
+                        && substr($newContent, -1) !== '.'
+                        && substr(strtolower($newContent), -1) !== 'e'
+                        && preg_match('`^(?:(?<![\.e])_[0-9][0-9e\.]*)+$`iD', $tokens[$i][1]) === 1))
+                    ) {
+                        $newContent .= $tokens[$i][1];
 
                         // Support floats.
-                        if ($tokens[$i][0] === T_STRING
-                            && substr(strtolower($tokens[$i][1]), -1) === 'e'
+                        if (substr(strtolower($tokens[$i][1]), -1) === 'e'
                             && ($tokens[($i + 1)] === '-'
                             || $tokens[($i + 1)] === '+')
                         ) {
@@ -1019,9 +1023,21 @@ protected function tokenize($string)
                     break;
                 }//end for
 
+                if ($newType === T_LNUMBER
+                    && ((stripos($newContent, '0x') === 0 && hexdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
+                    || (stripos($newContent, '0b') === 0 && bindec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
+                    || (stripos($newContent, '0x') !== 0
+                    && stripos($newContent, 'e') !== false || strpos($newContent, '.') !== false)
+                    || (strpos($newContent, '0') === 0 && stripos($newContent, '0x') !== 0
+                    && stripos($newContent, '0b') !== 0 && octdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
+                    || (strpos($newContent, '0') !== 0 && str_replace('_', '', $newContent) > PHP_INT_MAX))
+                ) {
+                    $newType = T_DNUMBER;
+                }
+
                 $newToken            = [];
                 $newToken['code']    = $newType;
-                $newToken['type']    = Util\Tokens::tokenName($token[0]);
+                $newToken['type']    = Util\Tokens::tokenName($newType);
                 $newToken['content'] = $newContent;
                 $finalTokens[$newStackPtr] = $newToken;
 

tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc

@@ -1,25 +1,82 @@
 <?php
 
+/*
+ * Numbers with PHP 7.4 underscore separators.
+ */
+
 /* testSimpleLNumber */
 $foo = 1_000_000_000;
 
 /* testSimpleDNumber */
-‪$foo = 107_925_284.88;
+$foo = 107_925_284.88;
 
 /* testFloat */
 $foo = 6.674_083e-11;
 
 /* testFloat2 */
 $foo = 6.674_083e+11;
 
+/* testFloat3 */
+$foo = 1_2.3_4e1_23;
+
 /* testHex */
 $foo = 0xCAFE_F00D;
 
 /* testHexMultiple */
 $foo = 0x42_72_6F_77_6E;
 
+/* testHexInt */
+$foo = 0x42_72_6F;
+
 /* testBinary */
 $foo = 0b0101_1111;
 
 /* testOctal */
 $foo = 0137_041;
+
+/* testIntMoreThanMax */
+$foo = 10_223_372_036_854_775_807;
+
+/*
+ * Invalid use of numeric separators. These should not be touched by the backfill.
+ */
+
+/* testInvalid1 */
+$a = 100_;   // trailing
+
+/* testInvalid2 */
+$a = 1__1;   // next to underscore
+
+/* testInvalid3 */
+$a = 1_.0;   // next to decimal point
+
+/* testInvalid4 */
+$a = 1._0;   // next to decimal point
+
+/* testInvalid5 */
+$a = 0x_123; // next to x
+
+/* testInvalid6 */
+$a = 0b_101; // next to b
+
+/* testInvalid7 */
+$a = 1_e2;   // next to e
+
+/* testInvalid8 */
+$a = 1e_2;   // next to e
+
+/* testInvalid9 */
+$testValue = 107_925_284 .88;
+
+/* testInvalid10 */
+$testValue = 107_925_284/*comment*/.88;
+
+/*
+ * Ensure that legitimate calculations are not touched by the backfill.
+ */
+
+/* testCalc1 */
+$a = 667_083 - 11; // Calculation.
+
+/* test Calc2 */
+$a = 6.674_08e3 + 11; // Calculation.