Tokenizer: add support for PHP8 attributes

package.xml

@@ -226,6 +226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
      <dir name="Tokenizer">
       <file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
       <file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
+      <file baseinstalldir="" name="AttributesTest.inc" role="test" />
+      <file baseinstalldir="" name="AttributesTest.php" role="test" />
       <file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
       <file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
       <file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
@@ -2138,6 +2140,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
@@ -2222,6 +2226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />

src/Files/File.php

@@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
                 T_SEMICOLON,
                 T_OPEN_CURLY_BRACKET,
                 T_CLOSE_CURLY_BRACKET,
+                T_ATTRIBUTE_END,
             ],
             ($stackPtr - 1)
         );

src/Tokenizers/PHP.php

@@ -903,6 +903,50 @@ protected function tokenize($string)
                 continue;
             }//end if
 
+            /*
+                PHP 8.0 Attributes
+            */
+
+            if (PHP_VERSION_ID < 80000
+                && $token[0] === T_COMMENT
+                && strpos($token[1], '#[') === 0
+            ) {
+                $subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
+                if ($subTokens !== null) {
+                    array_splice($tokens, $stackPtr, 1, $subTokens);
+                    $numTokens = count($tokens);
+
+                    $tokenIsArray = true;
+                    $token        = $tokens[$stackPtr];
+                } else {
+                    $token[0] = T_ATTRIBUTE;
+                }
+            }
+
+            if ($tokenIsArray === true
+                && $token[0] === T_ATTRIBUTE
+            ) {
+                // Go looking for the close bracket.
+                $bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']');
+
+                $newToken            = [];
+                $newToken['code']    = T_ATTRIBUTE;
+                $newToken['type']    = 'T_ATTRIBUTE';
+                $newToken['content'] = '#[';
+                $finalTokens[$newStackPtr] = $newToken;
+
+                $tokens[$bracketCloser]    = [];
+                $tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
+                $tokens[$bracketCloser][1] = ']';
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
+                }
+
+                $newStackPtr++;
+                continue;
+            }//end if
+
             /*
                 Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
                 token and ensure that the colon after it is always T_COLON.
@@ -1845,6 +1889,7 @@ function return types. We want to keep the parenthesis map clean,
                         T_CLASS                    => true,
                         T_EXTENDS                  => true,
                         T_IMPLEMENTS               => true,
+                        T_ATTRIBUTE                => true,
                         T_NEW                      => true,
                         T_CONST                    => true,
                         T_NS_SEPARATOR             => true,
@@ -2102,6 +2147,8 @@ protected function processAdditional()
             echo "\t*** START ADDITIONAL PHP PROCESSING ***".PHP_EOL;
         }
 
+        $this->createAttributesNestingMap();
+
         $numTokens = count($this->tokens);
         for ($i = ($numTokens - 1); $i >= 0; $i--) {
             // Check for any unset scope conditions due to alternate IF/ENDIF syntax.
@@ -3077,4 +3124,128 @@ public static function resolveSimpleToken($token)
     }//end resolveSimpleToken()
 
 
+    /**
+     * Finds a "closer" token (closing parenthesis or square bracket for example)
+     * Handle parenthesis balancing while searching for closing token
+     *
+     * @param array           $tokens       The list of tokens to iterate searching the closing token (as returned by token_get_all)
+     * @param int             $start        The starting position
+     * @param string|string[] $openerTokens The opening character
+     * @param string          $closerChar   The closing character
+     *
+     * @return int|null The position of the closing token, if found. NULL otherwise.
+     */
+    private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
+    {
+        $numTokens    = count($tokens);
+        $stack        = [0];
+        $closer       = null;
+        $openerTokens = (array) $openerTokens;
+
+        for ($x = $start; $x < $numTokens; $x++) {
+            if (in_array($tokens[$x], $openerTokens, true) === true
+                || (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true)
+            ) {
+                $stack[] = $x;
+            } else if ($tokens[$x] === $closerChar) {
+                array_pop($stack);
+                if (empty($stack) === true) {
+                    $closer = $x;
+                    break;
+                }
+            }
+        }
+
+        return $closer;
+
+    }//end findCloser()
+
+
+    /**
+     * PHP 8 attributes parser for PHP < 8
+     * Handles single-line and multiline attributes.
+     *
+     * @param array $tokens   The original array of tokens (as returned by token_get_all)
+     * @param int   $stackPtr The current position in token array
+     *
+     * @return array|null The array of parsed attribute tokens
+     */
+    private function parsePhpAttribute(array &$tokens, $stackPtr)
+    {
+
+        $token = $tokens[$stackPtr];
+
+        $commentBody = substr($token[1], 2);
+        $subTokens   = @token_get_all('<?php '.$commentBody);
+
+        foreach ($subTokens as $i => $subToken) {
+            if (is_array($subToken) === true
+                && $subToken[0] === T_COMMENT
+                && strpos($subToken[1], '#[') === 0
+            ) {
+                $reparsed = $this->parsePhpAttribute($subTokens, $i);
+                if ($reparsed !== null) {
+                    array_splice($subTokens, $i, 1, $reparsed);
+                } else {
+                    $subToken[0] = T_ATTRIBUTE;
+                }
+            }
+        }
+
+        array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
+
+        // Go looking for the close bracket.
+        $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+        if ($bracketCloser === null) {
+            $bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
+            if ($bracketCloser === null) {
+                return null;
+            }
+
+            $subTokens = array_merge($subTokens, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
+            array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
+        }
+
+        return $subTokens;
+
+    }//end parsePhpAttribute()
+
+
+    /**
+     * Creates a map for the attributes tokens that surround other tokens.
+     *
+     * @return void
+     */
+    private function createAttributesNestingMap()
+    {
+        $map = [];
+        for ($i = 0; $i < $this->numTokens; $i++) {
+            if (isset($this->tokens[$i]['attribute_opener']) === true
+                && $i === $this->tokens[$i]['attribute_opener']
+            ) {
+                if (empty($map) === false) {
+                    $this->tokens[$i]['nested_attributes'] = $map;
+                }
+
+                if (isset($this->tokens[$i]['attribute_closer']) === true) {
+                    $map[$this->tokens[$i]['attribute_opener']]
+                        = $this->tokens[$i]['attribute_closer'];
+                }
+            } else if (isset($this->tokens[$i]['attribute_closer']) === true
+                && $i === $this->tokens[$i]['attribute_closer']
+            ) {
+                array_pop($map);
+                if (empty($map) === false) {
+                    $this->tokens[$i]['nested_attributes'] = $map;
+                }
+            } else {
+                if (empty($map) === false) {
+                    $this->tokens[$i]['nested_attributes'] = $map;
+                }
+            }//end if
+        }//end for
+
+    }//end createAttributesNestingMap()
+
+
 }//end class

src/Tokenizers/Tokenizer.php

@@ -740,6 +740,40 @@ private function createTokenMap()
                     $this->tokens[$i]['parenthesis_closer']      = $i;
                     $this->tokens[$opener]['parenthesis_closer'] = $i;
                 }//end if
+            } else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
+                $openers[] = $i;
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo str_repeat("\t", count($openers));
+                    echo "=> Found attribute opener at $i".PHP_EOL;
+                }
+
+                $this->tokens[$i]['attribute_opener'] = $i;
+                $this->tokens[$i]['attribute_closer'] = null;
+            } else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) {
+                $numOpeners = count($openers);
+                if ($numOpeners !== 0) {
+                    $opener = array_pop($openers);
+                    if (isset($this->tokens[$opener]['attribute_opener']) === true) {
+                        $this->tokens[$opener]['attribute_closer'] = $i;
+
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            echo str_repeat("\t", (count($openers) + 1));
+                            echo "=> Found attribute closer at $i for $opener".PHP_EOL;
+                        }
+
+                        for ($x = ($opener + 1); $x <= $i; ++$x) {
+                            if (isset($this->tokens[$x]['attribute_closer']) === true) {
+                                continue;
+                            }
+
+                            $this->tokens[$x]['attribute_opener'] = $opener;
+                            $this->tokens[$x]['attribute_closer'] = $i;
+                        }
+                    } else if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                        echo str_repeat("\t", (count($openers) + 1));
+                        echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL;
+                    }
+                }//end if
             }//end if
 
             /*

src/Util/Tokens.php

@@ -79,6 +79,7 @@
 define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
 define('T_MATCH_ARROW', 'PHPCS_T_MATCH_ARROW');
 define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
+define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');
 
 // Some PHP 5.5 tokens, replicated for lower versions.
 if (defined('T_FINALLY') === false) {
@@ -149,6 +150,10 @@
     define('T_MATCH', 'PHPCS_T_MATCH');
 }
 
+if (defined('T_ATTRIBUTE') === false) {
+    define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
+}
+
 // Tokens used for parsing doc blocks.
 define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
 define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

tests/Core/File/GetMemberPropertiesTest.inc

@@ -240,3 +240,19 @@ $anon = class() {
     // Intentional fatal error - duplicate types are not allowed in union types, but that's not the concern of the method.
     public int |string| /*comment*/ INT $duplicateTypeInUnion;
 };
+
+$anon = class {
+    /* testPHP8PropertySingleAttribute */
+    #[PropertyWithAttribute]
+    public string $foo;
+
+    /* testPHP8PropertyMultipleAttributes */
+    #[PropertyWithAttribute(foo: 'bar'), MyAttribute]
+    protected ?int|float $bar;
+
+    /* testPHP8PropertyMultilineAttribute */
+    #[
+        PropertyWithAttribute(/* comment */ 'baz')
+    ]
+    private mixed $baz;
+};

tests/Core/File/GetMemberPropertiesTest.php

@@ -610,6 +610,36 @@ public function dataGetMemberProperties()
                     'nullable_type'   => false,
                 ],
             ],
+            [
+                '/* testPHP8PropertySingleAttribute */',
+                [
+                    'scope'           => 'public',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => 'string',
+                    'nullable_type'   => false,
+                ],
+            ],
+            [
+                '/* testPHP8PropertyMultipleAttributes */',
+                [
+                    'scope'           => 'protected',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => '?int|float',
+                    'nullable_type'   => true,
+                ],
+            ],
+            [
+                '/* testPHP8PropertyMultilineAttribute */',
+                [
+                    'scope'           => 'private',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => 'mixed',
+                    'nullable_type'   => false,
+                ],
+            ],
         ];
 
     }//end dataGetMemberProperties()