Skip to content

Commit 868be7e

Browse files
alekittoalekitto
committed
handled nested attributes tokenization
1 parent 2f92a4d commit 868be7e

File tree

4 files changed

+177
-24
lines changed

4 files changed

+177
-24
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -927,7 +927,7 @@ protected function tokenize($string)
927927
&& $token[0] === T_ATTRIBUTE
928928
) {
929929
// Go looking for the close bracket.
930-
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), '[', ']');
930+
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']');
931931

932932
$newToken = [];
933933
$newToken['code'] = T_ATTRIBUTE;
@@ -3126,20 +3126,24 @@ public static function resolveSimpleToken($token)
31263126
* Finds a "closer" token (closing parenthesis or square bracket for example)
31273127
* Handle parenthesis balancing while searching for closing token
31283128
*
3129-
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
3130-
* @param int $start The starting position
3131-
* @param string $openerChar The opening character
3132-
* @param string $closerChar The closing character
3129+
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
3130+
* @param int $start The starting position
3131+
* @param string|string[] $openerTokens The opening character
3132+
* @param string $closerChar The closing character
31333133
*
31343134
* @return int|null The position of the closing token, if found. NULL otherwise.
31353135
*/
3136-
private function findCloser(array &$tokens, $start, $openerChar, $closerChar)
3136+
private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
31373137
{
3138-
$numTokens = count($tokens);
3139-
$stack = [0];
3140-
$closer = null;
3138+
$numTokens = count($tokens);
3139+
$stack = [0];
3140+
$closer = null;
3141+
$openerTokens = (array) $openerTokens;
3142+
31413143
for ($x = $start; $x < $numTokens; $x++) {
3142-
if ($tokens[$x] === $openerChar) {
3144+
if (in_array($tokens[$x], $openerTokens, true) === true
3145+
|| (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true)
3146+
) {
31433147
$stack[] = $x;
31443148
} else if ($tokens[$x] === $closerChar) {
31453149
array_pop($stack);
@@ -3171,6 +3175,21 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)
31713175

31723176
$commentBody = substr($token[1], 2);
31733177
$subTokens = @token_get_all('<?php '.$commentBody);
3178+
3179+
foreach ($subTokens as $i => $subToken) {
3180+
if (is_array($subToken) === true
3181+
&& $subToken[0] === T_COMMENT
3182+
&& strpos($subToken[1], '#[') === 0
3183+
) {
3184+
$reparsed = $this->parsePhpAttribute($subTokens, $i);
3185+
if ($reparsed !== null) {
3186+
array_splice($subTokens, $i, 1, $reparsed);
3187+
} else {
3188+
$subToken[0] = T_ATTRIBUTE;
3189+
}
3190+
}
3191+
}
3192+
31743193
array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
31753194

31763195
// Go looking for the close bracket.

‎src/Tokenizers/Tokenizer.php‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -741,24 +741,39 @@ private function createTokenMap()
741741
$this->tokens[$opener]['parenthesis_closer'] = $i;
742742
}//end if
743743
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
744-
$found = null;
745-
$numTokens = count($this->tokens);
746-
for ($x = ($i + 1); $x < $numTokens; $x++) {
747-
if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) {
748-
$found = $x;
749-
break;
750-
}
744+
$openers[] = $i;
745+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
746+
echo str_repeat("\t", count($openers));
747+
echo "=> Found attribute opener at $i".PHP_EOL;
751748
}
752749

753750
$this->tokens[$i]['attribute_opener'] = $i;
754-
$this->tokens[$i]['attribute_closer'] = $found;
751+
$this->tokens[$i]['attribute_closer'] = null;
752+
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) {
753+
$numOpeners = count($openers);
754+
if ($numOpeners !== 0) {
755+
$opener = array_pop($openers);
756+
if (isset($this->tokens[$opener]['attribute_opener']) === true) {
757+
$this->tokens[$opener]['attribute_closer'] = $i;
755758

756-
if ($found !== null) {
757-
for ($x = ($i + 1); $x <= $found; ++$x) {
758-
$this->tokens[$x]['attribute_opener'] = $i;
759-
$this->tokens[$x]['attribute_closer'] = $found;
759+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
760+
echo str_repeat("\t", (count($openers) + 1));
761+
echo "=> Found attribute closer at $i for $opener".PHP_EOL;
762+
}
763+
764+
for ($x = ($opener + 1); $x <= $i; ++$x) {
765+
if (isset($this->tokens[$x]['attribute_closer']) === true) {
766+
continue;
767+
}
768+
769+
$this->tokens[$x]['attribute_opener'] = $opener;
770+
$this->tokens[$x]['attribute_closer'] = $i;
771+
}
772+
} else if (PHP_CODESNIFFER_VERBOSITY > 1) {
773+
echo str_repeat("\t", (count($openers) + 1));
774+
echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL;
760775
}
761-
}
776+
}//end if
762777
}//end if
763778

764779
/*

‎tests/Core/Tokenizer/AttributesTest.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithPa
6464
#[Boo\QualifiedName, \Foo\FullyQualifiedName('foo')]
6565
function fqcn_attrebute_test() {}
6666

67+
/* testNestedAttributes */
68+
#[Boo\QualifiedName(fn (#[AttributeOne('boo')] $value) => (string) $value)]
69+
function nested_attributes_test() {}
70+
6771
/* testMultilineAttributesOnParameter */
6872
function multiline_attributes_on_parameter_test(#[
6973
AttributeWithParams(

‎tests/Core/Tokenizer/AttributesTest.php‎

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ public function testAttributeAndLineComment()
294294
* @param string $testMarker The comment which prefaces the target token in the test file.
295295
* @param int $position The token position (starting from T_FUNCTION) of T_ATTRIBUTE token.
296296
* @param int $length The number of tokens between opener and closer.
297+
* @param array $tokenCodes The codes of tokens inside the attributes.
297298
*
298299
* @dataProvider dataAttributeOnParameters
299300
*
@@ -303,7 +304,7 @@ public function testAttributeAndLineComment()
303304
*
304305
* @return void
305306
*/
306-
public function testAttributeOnParameters($testMarker, $position, $length)
307+
public function testAttributeOnParameters($testMarker, $position, $length, array $tokenCodes)
307308
{
308309
$tokens = self::$phpcsFile->getTokens();
309310

@@ -312,6 +313,7 @@ public function testAttributeOnParameters($testMarker, $position, $length)
312313

313314
$this->assertSame(T_ATTRIBUTE, $tokens[$attribute]['code']);
314315
$this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
316+
315317
$this->assertSame(($attribute + $length), $tokens[$attribute]['attribute_closer']);
316318

317319
$closer = $tokens[$attribute]['attribute_closer'];
@@ -322,6 +324,18 @@ public function testAttributeOnParameters($testMarker, $position, $length)
322324
$this->assertSame(T_VARIABLE, $tokens[($closer + 4)]['code']);
323325
$this->assertSame('$param', $tokens[($closer + 4)]['content']);
324326

327+
$map = array_map(
328+
function ($token) use ($attribute, $length) {
329+
$this->assertArrayHasKey('attribute_closer', $token);
330+
$this->assertSame(($attribute + $length), $token['attribute_closer']);
331+
332+
return $token['code'];
333+
},
334+
array_slice($tokens, ($attribute + 1), ($length - 1))
335+
);
336+
337+
$this->assertSame($tokenCodes, $map);
338+
325339
}//end testAttributeOnParameters()
326340

327341

@@ -339,16 +353,42 @@ public function dataAttributeOnParameters()
339353
'/* testSingleAttributeOnParameter */',
340354
4,
341355
2,
356+
[T_STRING],
342357
],
343358
[
344359
'/* testMultipleAttributesOnParameter */',
345360
4,
346361
10,
362+
[
363+
T_STRING,
364+
T_COMMA,
365+
T_WHITESPACE,
366+
T_STRING,
367+
T_OPEN_PARENTHESIS,
368+
T_COMMENT,
369+
T_WHITESPACE,
370+
T_CONSTANT_ENCAPSED_STRING,
371+
T_CLOSE_PARENTHESIS,
372+
],
347373
],
348374
[
349375
'/* testMultilineAttributesOnParameter */',
350376
4,
351377
13,
378+
[
379+
T_WHITESPACE,
380+
T_WHITESPACE,
381+
T_STRING,
382+
T_OPEN_PARENTHESIS,
383+
T_WHITESPACE,
384+
T_WHITESPACE,
385+
T_CONSTANT_ENCAPSED_STRING,
386+
T_WHITESPACE,
387+
T_WHITESPACE,
388+
T_CLOSE_PARENTHESIS,
389+
T_WHITESPACE,
390+
T_WHITESPACE,
391+
],
352392
],
353393
];
354394

@@ -376,4 +416,79 @@ public function testInvalidAttribute()
376416
}//end testInvalidAttribute()
377417

378418

419+
/**
420+
* Test that nested attributes are parsed correctly.
421+
*
422+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
423+
* @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser
424+
* @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
425+
*
426+
* @return void
427+
*/
428+
public function testNestedAttributes()
429+
{
430+
$tokens = self::$phpcsFile->getTokens();
431+
$tokenCodes = [
432+
T_STRING,
433+
T_NS_SEPARATOR,
434+
T_STRING,
435+
T_OPEN_PARENTHESIS,
436+
T_FN,
437+
T_WHITESPACE,
438+
T_OPEN_PARENTHESIS,
439+
T_ATTRIBUTE,
440+
T_STRING,
441+
T_OPEN_PARENTHESIS,
442+
T_CONSTANT_ENCAPSED_STRING,
443+
T_CLOSE_PARENTHESIS,
444+
T_ATTRIBUTE_END,
445+
T_WHITESPACE,
446+
T_VARIABLE,
447+
T_CLOSE_PARENTHESIS,
448+
T_WHITESPACE,
449+
T_FN_ARROW,
450+
T_WHITESPACE,
451+
T_STRING_CAST,
452+
T_WHITESPACE,
453+
T_VARIABLE,
454+
T_CLOSE_PARENTHESIS,
455+
];
456+
457+
$attribute = $this->getTargetToken('/* testNestedAttributes */', T_ATTRIBUTE);
458+
$this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
459+
460+
$closer = $tokens[$attribute]['attribute_closer'];
461+
$this->assertSame(($attribute + 24), $closer);
462+
463+
$this->assertSame(T_ATTRIBUTE_END, $tokens[$closer]['code']);
464+
465+
$this->assertSame($tokens[$attribute]['attribute_opener'], $tokens[$closer]['attribute_opener']);
466+
$this->assertSame($tokens[$attribute]['attribute_closer'], $tokens[$closer]['attribute_closer']);
467+
468+
$test = function (array $tokens, $length) use ($attribute) {
469+
foreach ($tokens as $token) {
470+
$this->assertArrayHasKey('attribute_closer', $token);
471+
$this->assertSame(($attribute + $length), $token['attribute_closer']);
472+
}
473+
};
474+
475+
$test(array_slice($tokens, ($attribute + 1), 7), 24);
476+
477+
// Length here is 8 (nested attribute offset) + 5 (real length).
478+
$test(array_slice($tokens, ($attribute + 8), 6), 8 + 5);
479+
480+
$test(array_slice($tokens, ($attribute + 14), 11), 24);
481+
482+
$map = array_map(
483+
static function ($token) {
484+
return $token['code'];
485+
},
486+
array_slice($tokens, ($attribute + 1), 23)
487+
);
488+
489+
$this->assertSame($tokenCodes, $map);
490+
491+
}//end testNestedAttributes()
492+
493+
379494
}//end class

0 commit comments

Comments
 (0)