Skip to content

Commit e0152a7

Browse files
alekittoalekitto
committed
tokenizer: add support for php8 attributes
1 parent 18c27ed commit e0152a7

File tree

9 files changed

+593
-0
lines changed

9 files changed

+593
-0
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
226226
<dir name="Tokenizer">
227227
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
228228
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
229+
<file baseinstalldir="" name="AttributesTest.inc" role="test" />
230+
<file baseinstalldir="" name="AttributesTest.php" role="test" />
229231
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
230232
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
231233
<file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
@@ -2138,6 +2140,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21382140
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
21392141
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
21402142
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2143+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2144+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
21412145
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
21422146
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
21432147
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
@@ -2222,6 +2226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
22222226
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
22232227
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
22242228
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2229+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2230+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
22252231
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
22262232
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
22272233
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />

‎src/Files/File.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
18131813
T_SEMICOLON,
18141814
T_OPEN_CURLY_BRACKET,
18151815
T_CLOSE_CURLY_BRACKET,
1816+
T_ATTRIBUTE_END,
18161817
],
18171818
($stackPtr - 1)
18181819
);

‎src/Tokenizers/PHP.php‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,50 @@ protected function tokenize($string)
903903
continue;
904904
}//end if
905905

906+
/*
907+
PHP 8.0 Attributes
908+
*/
909+
910+
if (PHP_VERSION_ID < 80000
911+
&& $token[0] === T_COMMENT
912+
&& strpos($token[1], '#[') === 0
913+
) {
914+
$subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
915+
if ($subTokens !== null) {
916+
array_splice($tokens, $stackPtr, 1, $subTokens);
917+
$numTokens = count($tokens);
918+
919+
$tokenIsArray = true;
920+
$token = $tokens[$stackPtr];
921+
} else {
922+
$token[0] = T_ATTRIBUTE;
923+
}
924+
}
925+
926+
if ($tokenIsArray === true
927+
&& $token[0] === T_ATTRIBUTE
928+
) {
929+
// Go looking for the close bracket.
930+
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), '[', ']');
931+
932+
$newToken = [];
933+
$newToken['code'] = T_ATTRIBUTE;
934+
$newToken['type'] = 'T_ATTRIBUTE';
935+
$newToken['content'] = '#[';
936+
$finalTokens[$newStackPtr] = $newToken;
937+
938+
$tokens[$bracketCloser] = [];
939+
$tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
940+
$tokens[$bracketCloser][1] = ']';
941+
942+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
943+
echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
944+
}
945+
946+
$newStackPtr++;
947+
continue;
948+
}//end if
949+
906950
/*
907951
Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
908952
token and ensure that the colon after it is always T_COLON.
@@ -1845,6 +1889,7 @@ function return types. We want to keep the parenthesis map clean,
18451889
T_CLASS => true,
18461890
T_EXTENDS => true,
18471891
T_IMPLEMENTS => true,
1892+
T_ATTRIBUTE => true,
18481893
T_NEW => true,
18491894
T_CONST => true,
18501895
T_NS_SEPARATOR => true,
@@ -3077,4 +3122,72 @@ public static function resolveSimpleToken($token)
30773122
}//end resolveSimpleToken()
30783123

30793124

3125+
/**
3126+
* Finds a "closer" token (closing parenthesis or square bracket for example)
3127+
* Handle parenthesis balancing while searching for closing token
3128+
*
3129+
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
3130+
* @param int $start The starting position
3131+
* @param string $openerChar The opening character
3132+
* @param string $closerChar The closing character
3133+
*
3134+
* @return int|null The position of the closing token, if found. NULL otherwise.
3135+
*/
3136+
private function findCloser(array &$tokens, $start, $openerChar, $closerChar)
3137+
{
3138+
$numTokens = count($tokens);
3139+
$stack = [0];
3140+
$closer = null;
3141+
for ($x = $start; $x < $numTokens; $x++) {
3142+
if ($tokens[$x] === $openerChar) {
3143+
$stack[] = $x;
3144+
} else if ($tokens[$x] === $closerChar) {
3145+
array_pop($stack);
3146+
if (empty($stack) === true) {
3147+
$closer = $x;
3148+
break;
3149+
}
3150+
}
3151+
}
3152+
3153+
return $closer;
3154+
3155+
}//end findCloser()
3156+
3157+
3158+
/**
3159+
* PHP 8 attributes parser for PHP < 8
3160+
* Handles single-line and multiline attributes.
3161+
*
3162+
* @param array $tokens The original array of tokens (as returned by token_get_all)
3163+
* @param int $stackPtr The current position in token array
3164+
*
3165+
* @return array|null The array of parsed attribute tokens
3166+
*/
3167+
private function parsePhpAttribute(array &$tokens, $stackPtr)
3168+
{
3169+
3170+
$token = $tokens[$stackPtr];
3171+
3172+
$commentBody = substr($token[1], 2);
3173+
$subTokens = @token_get_all('<?php '.$commentBody);
3174+
array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
3175+
3176+
// Go looking for the close bracket.
3177+
$bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
3178+
if ($bracketCloser === null) {
3179+
$bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
3180+
if ($bracketCloser === null) {
3181+
return null;
3182+
}
3183+
3184+
array_splice($subTokens, count($subTokens), 0, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
3185+
array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
3186+
}
3187+
3188+
return $subTokens;
3189+
3190+
}//end parsePhpAttribute()
3191+
3192+
30803193
}//end class

‎src/Tokenizers/Tokenizer.php‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,23 @@ private function createTokenMap()
740740
$this->tokens[$i]['parenthesis_closer'] = $i;
741741
$this->tokens[$opener]['parenthesis_closer'] = $i;
742742
}//end if
743+
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
744+
$found = null;
745+
$numTokens = count($this->tokens);
746+
for ($x = ($i + 1); $x < $numTokens; $x++) {
747+
if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) {
748+
$found = $x;
749+
break;
750+
}
751+
}
752+
753+
$this->tokens[$i]['attribute_opener'] = $i;
754+
$this->tokens[$i]['attribute_closer'] = $found;
755+
756+
if ($found !== null) {
757+
$this->tokens[$found]['attribute_opener'] = $i;
758+
$this->tokens[$found]['attribute_closer'] = $found;
759+
}
743760
}//end if
744761

745762
/*

‎src/Util/Tokens.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
8080
define('T_MATCH_ARROW', 'PHPCS_T_MATCH_ARROW');
8181
define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
82+
define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');
8283

8384
// Some PHP 5.5 tokens, replicated for lower versions.
8485
if (defined('T_FINALLY') === false) {
@@ -149,6 +150,10 @@
149150
define('T_MATCH', 'PHPCS_T_MATCH');
150151
}
151152

153+
if (defined('T_ATTRIBUTE') === false) {
154+
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
155+
}
156+
152157
// Tokens used for parsing doc blocks.
153158
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
154159
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/File/GetMemberPropertiesTest.inc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,3 +240,19 @@ $anon = class() {
240240
// Intentional fatal error - duplicate types are not allowed in union types, but that's not the concern of the method.
241241
public int |string| /*comment*/ INT $duplicateTypeInUnion;
242242
};
243+
244+
$anon = class {
245+
/* testPHP8PropertySingleAttribute */
246+
#[PropertyWithAttribute]
247+
public string $foo;
248+
249+
/* testPHP8PropertyMultipleAttributes */
250+
#[PropertyWithAttribute(foo: 'bar'), MyAttribute]
251+
protected ?int|float $bar;
252+
253+
/* testPHP8PropertyMultilineAttribute */
254+
#[
255+
PropertyWithAttribute(/* comment */ 'baz')
256+
]
257+
private mixed $baz;
258+
};

‎tests/Core/File/GetMemberPropertiesTest.php‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,36 @@ public function dataGetMemberProperties()
610610
'nullable_type' => false,
611611
],
612612
],
613+
[
614+
'/* testPHP8PropertySingleAttribute */',
615+
[
616+
'scope' => 'public',
617+
'scope_specified' => true,
618+
'is_static' => false,
619+
'type' => 'string',
620+
'nullable_type' => false,
621+
],
622+
],
623+
[
624+
'/* testPHP8PropertyMultipleAttributes */',
625+
[
626+
'scope' => 'protected',
627+
'scope_specified' => true,
628+
'is_static' => false,
629+
'type' => '?int|float',
630+
'nullable_type' => true,
631+
],
632+
],
633+
[
634+
'/* testPHP8PropertyMultilineAttribute */',
635+
[
636+
'scope' => 'private',
637+
'scope_specified' => true,
638+
'is_static' => false,
639+
'type' => 'mixed',
640+
'nullable_type' => false,
641+
],
642+
],
613643
];
614644

615645
}//end dataGetMemberProperties()

‎tests/Core/Tokenizer/AttributesTest.inc‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?php
2+
3+
/* testAttribute */
4+
#[Attribute]
5+
class CustomAttribute {}
6+
7+
/* testAttributeWithParams */
8+
#[Attribute(Attribute::TARGET_CLASS)]
9+
class SecondCustomAttribute {}
10+
11+
/* testAttributeWithNamedParam */
12+
#[Attribute(flags: Attribute::TARGET_ALL)]
13+
class AttributeWithParams {
14+
public function __construct($foo, array $bar) {}
15+
}
16+
17+
/* testAttributeOnFunction */
18+
#[CustomAttribute]
19+
function attribute_on_function_test() {}
20+
21+
/* testAttributeOnFunctionWithParams */
22+
#[AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
23+
function attribute_with_params_on_function_test() {}
24+
25+
/* testTwoAttributeOnTheSameLine */
26+
#[CustomAttribute] #[AttributeWithParams('foo')]
27+
function two_attribute_on_same_line_test() {}
28+
29+
/* testAttributeAndCommentOnTheSameLine */
30+
#[CustomAttribute] // This is a comment
31+
function attribute_and_line_comment_on_same_line_test() {}
32+
33+
/* testAttributeGrouping */
34+
#[CustomAttribute, AttributeWithParams('foo'), AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
35+
function attribute_grouping_test() {}
36+
37+
/* testAttributeMultiline */
38+
#[
39+
CustomAttribute,
40+
AttributeWithParams('foo'),
41+
AttributeWithParams('foo', bar: ['bar' => 'foobar'])
42+
]
43+
function attribute_multiline_test() {}
44+
45+
/* testAttributeMultilineWithComment */
46+
#[
47+
CustomAttribute, // comment
48+
AttributeWithParams(/* another comment */ 'foo'),
49+
AttributeWithParams('foo', bar: ['bar' => 'foobar'])
50+
]
51+
function attribute_multiline_with_comment_test() {}
52+
53+
/* testSingleAttributeOnParameter */
54+
function single_attribute_on_parameter_test(#[ParamAttribute] int $param) {}
55+
56+
/* testMultipleAttributesOnParameter */
57+
function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithParams(/* another comment */ 'foo')] int $param) {}
58+
59+
/* testMultilineAttributesOnParameter */
60+
function multiline_attributes_on_parameter_test(#[
61+
AttributeWithParams(
62+
'foo'
63+
)
64+
] int $param) {}
65+
66+
/* testInvalidAttribute */
67+
#[ThisIsNotAnAttribute
68+
function invalid_attribute_test() {}
69+

0 commit comments

Comments
 (0)