Skip to content

Commit 6525028

Browse files
gsherwoodgsherwood
committed
Merge branch 'feature/3041-backport-old-identifier-tokenization-php-8' of https://github.com/jrfnl/PHP_CodeSniffer 
2 parents 4201fd8 + ba63323 commit 6525028

File tree

5 files changed

+1562
-9
lines changed

5 files changed

+1562
-9
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
128128
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
129129
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
130130
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
131+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
132+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
131133
</dir>
132134
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
133135
<file baseinstalldir="" name="AllTests.php" role="test" />
@@ -2004,6 +2006,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20042006
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
20052007
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
20062008
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2009+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2010+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20072011
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20082012
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20092013
</filelist>
@@ -2065,6 +2069,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20652069
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
20662070
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
20672071
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2072+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2073+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20682074
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20692075
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20702076
<ignore name="bin/phpcs.bat" />

‎src/Tokenizers/PHP.php‎

Lines changed: 100 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -815,6 +815,81 @@ protected function tokenize($string)
815815
continue;
816816
}//end if
817817

818+
/*
819+
As of PHP 8.0 fully qualified, partially qualified and namespace relative
820+
identifier names are tokenized differently.
821+
This "undoes" the new tokenization so the tokenization will be the same in
822+
in PHP 5, 7 and 8.
823+
*/
824+
825+
if (PHP_VERSION_ID >= 80000
826+
&& $tokenIsArray === true
827+
&& ($token[0] === T_NAME_QUALIFIED
828+
|| $token[0] === T_NAME_FULLY_QUALIFIED
829+
|| $token[0] === T_NAME_RELATIVE)
830+
) {
831+
$name = $token[1];
832+
833+
if ($token[0] === T_NAME_FULLY_QUALIFIED) {
834+
$newToken = [];
835+
$newToken['code'] = T_NS_SEPARATOR;
836+
$newToken['type'] = 'T_NS_SEPARATOR';
837+
$newToken['content'] = '\\';
838+
$finalTokens[$newStackPtr] = $newToken;
839+
++$newStackPtr;
840+
841+
$name = ltrim($name, '\\');
842+
}
843+
844+
if ($token[0] === T_NAME_RELATIVE) {
845+
$newToken = [];
846+
$newToken['code'] = T_NAMESPACE;
847+
$newToken['type'] = 'T_NAMESPACE';
848+
$newToken['content'] = substr($name, 0, 9);
849+
$finalTokens[$newStackPtr] = $newToken;
850+
++$newStackPtr;
851+
852+
$newToken = [];
853+
$newToken['code'] = T_NS_SEPARATOR;
854+
$newToken['type'] = 'T_NS_SEPARATOR';
855+
$newToken['content'] = '\\';
856+
$finalTokens[$newStackPtr] = $newToken;
857+
++$newStackPtr;
858+
859+
$name = substr($name, 10);
860+
}
861+
862+
$parts = explode('\\', $name);
863+
$partCount = count($parts);
864+
$lastPart = ($partCount - 1);
865+
866+
foreach ($parts as $i => $part) {
867+
$newToken = [];
868+
$newToken['code'] = T_STRING;
869+
$newToken['type'] = 'T_STRING';
870+
$newToken['content'] = $part;
871+
$finalTokens[$newStackPtr] = $newToken;
872+
++$newStackPtr;
873+
874+
if ($i !== $lastPart) {
875+
$newToken = [];
876+
$newToken['code'] = T_NS_SEPARATOR;
877+
$newToken['type'] = 'T_NS_SEPARATOR';
878+
$newToken['content'] = '\\';
879+
$finalTokens[$newStackPtr] = $newToken;
880+
++$newStackPtr;
881+
}
882+
}
883+
884+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
885+
$type = Util\Tokens::tokenName($token[0]);
886+
$content = Util\Common::prepareForOutput($token[1]);
887+
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
888+
}
889+
890+
continue;
891+
}//end if
892+
818893
/*
819894
Before PHP 7.0, the "yield from" was tokenized as
820895
T_YIELD, T_WHITESPACE and T_STRING. So look for
@@ -1131,7 +1206,7 @@ protected function tokenize($string)
11311206
* Check if the next non-empty token is one of the tokens which can be used
11321207
* in type declarations. If not, it's definitely a ternary.
11331208
* At this point, the only token types which need to be taken into consideration
1134-
* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
1209+
* as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
11351210
*/
11361211

11371212
$lastRelevantNonEmpty = null;
@@ -1148,6 +1223,9 @@ protected function tokenize($string)
11481223
}
11491224

11501225
if ($tokenType === T_STRING
1226+
|| $tokenType === T_NAME_FULLY_QUALIFIED
1227+
|| $tokenType === T_NAME_RELATIVE
1228+
|| $tokenType === T_NAME_QUALIFIED
11511229
|| $tokenType === T_ARRAY
11521230
|| $tokenType === T_NS_SEPARATOR
11531231
) {
@@ -1159,7 +1237,10 @@ protected function tokenize($string)
11591237
&& isset($lastRelevantNonEmpty) === false)
11601238
|| ($lastRelevantNonEmpty === T_ARRAY
11611239
&& $tokenType === '(')
1162-
|| ($lastRelevantNonEmpty === T_STRING
1240+
|| (($lastRelevantNonEmpty === T_STRING
1241+
|| $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
1242+
|| $lastRelevantNonEmpty === T_NAME_RELATIVE
1243+
|| $lastRelevantNonEmpty === T_NAME_QUALIFIED)
11631244
&& ($tokenType === T_DOUBLE_COLON
11641245
|| $tokenType === '('
11651246
|| $tokenType === ':'))
@@ -1304,6 +1385,10 @@ protected function tokenize($string)
13041385
tokenized as T_STRING even if it appears to be a different token,
13051386
such as when writing code like: function default(): foo
13061387
so go forward and change the token type before it is processed.
1388+
1389+
Note: this should not be done for `function Level\Name` within a
1390+
group use statement for the PHP 8 identifier name tokens as it
1391+
would interfere with the re-tokenization of those.
13071392
*/
13081393

13091394
if ($tokenIsArray === true
@@ -1321,7 +1406,10 @@ protected function tokenize($string)
13211406
}
13221407
}
13231408

1324-
if ($x < $numTokens && is_array($tokens[$x]) === true) {
1409+
if ($x < $numTokens
1410+
&& is_array($tokens[$x]) === true
1411+
&& $tokens[$x][0] !== T_NAME_QUALIFIED
1412+
) {
13251413
if (PHP_CODESNIFFER_VERBOSITY > 1) {
13261414
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
13271415
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
@@ -1377,12 +1465,15 @@ function return types. We want to keep the parenthesis map clean,
13771465
&& $tokens[$x] === ':'
13781466
) {
13791467
$allowed = [
1380-
T_STRING => T_STRING,
1381-
T_ARRAY => T_ARRAY,
1382-
T_CALLABLE => T_CALLABLE,
1383-
T_SELF => T_SELF,
1384-
T_PARENT => T_PARENT,
1385-
T_NS_SEPARATOR => T_NS_SEPARATOR,
1468+
T_STRING => T_STRING,
1469+
T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
1470+
T_NAME_RELATIVE => T_NAME_RELATIVE,
1471+
T_NAME_QUALIFIED => T_NAME_QUALIFIED,
1472+
T_ARRAY => T_ARRAY,
1473+
T_CALLABLE => T_CALLABLE,
1474+
T_SELF => T_SELF,
1475+
T_PARENT => T_PARENT,
1476+
T_NS_SEPARATOR => T_NS_SEPARATOR,
13861477
];
13871478

13881479
$allowed += Util\Tokens::$emptyTokens;

‎src/Util/Tokens.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,18 @@
129129
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
130130
}
131131

132+
if (defined('T_NAME_QUALIFIED') === false) {
133+
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
134+
}
135+
136+
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
137+
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
138+
}
139+
140+
if (defined('T_NAME_RELATIVE') === false) {
141+
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
142+
}
143+
132144
// Tokens used for parsing doc blocks.
133145
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
134146
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
<?php
2+
3+
/* testNamespaceDeclaration */
4+
namespace Package;
5+
6+
/* testNamespaceDeclarationWithLevels */
7+
namespace Vendor\SubLevel\Domain;
8+
9+
/* testUseStatement */
10+
use ClassName;
11+
12+
/* testUseStatementWithLevels */
13+
use Vendor\Level\Domain;
14+
15+
/* testFunctionUseStatement */
16+
use function function_name;
17+
18+
/* testFunctionUseStatementWithLevels */
19+
use function Vendor\Level\function_in_ns;
20+
21+
/* testConstantUseStatement */
22+
use const CONSTANT_NAME;
23+
24+
/* testConstantUseStatementWithLevels */
25+
use const Vendor\Level\OTHER_CONSTANT;
26+
27+
/* testMultiUseUnqualified */
28+
use UnqualifiedClassName,
29+
/* testMultiUsePartiallyQualified */
30+
Sublevel\PartiallyClassName;
31+
32+
/* testGroupUseStatement */
33+
use Vendor\Level\{
34+
AnotherDomain,
35+
function function_grouped,
36+
const CONSTANT_GROUPED,
37+
Sub\YetAnotherDomain,
38+
function SubLevelA\function_grouped_too,
39+
const SubLevelB\CONSTANT_GROUPED_TOO,
40+
};
41+
42+
/* testClassName */
43+
class MyClass
44+
/* testExtendedFQN */
45+
extends \Vendor\Level\FQN
46+
/* testImplementsRelative */
47+
implements namespace\Name,
48+
/* testImplementsFQN */
49+
\Fully\Qualified,
50+
/* testImplementsUnqualified */
51+
Unqualified,
52+
/* testImplementsPartiallyQualified */
53+
Sub\Level\Name
54+
{
55+
/* testFunctionName */
56+
public function function_name(
57+
/* testTypeDeclarationRelative */
58+
?namespace\Name|object $paramA,
59+
60+
/* testTypeDeclarationFQN */
61+
\Fully\Qualified\Name $paramB,
62+
63+
/* testTypeDeclarationUnqualified */
64+
Unqualified|false $paramC,
65+
66+
/* testTypeDeclarationPartiallyQualified */
67+
?Sublevel\Name $paramD,
68+
69+
/* testReturnTypeFQN */
70+
) : ?\Name {
71+
72+
try {
73+
/* testFunctionCallRelative */
74+
echo NameSpace\function_name();
75+
76+
/* testFunctionCallFQN */
77+
echo \Vendor\Package\function_name();
78+
79+
/* testFunctionCallUnqualified */
80+
echo function_name();
81+
82+
/* testFunctionPartiallyQualified */
83+
echo Level\function_name();
84+
85+
/* testCatchRelative */
86+
} catch (namespace\SubLevel\Exception $e) {
87+
88+
/* testCatchFQN */
89+
} catch (\Exception $e) {
90+
91+
/* testCatchUnqualified */
92+
} catch (Exception $e) {
93+
94+
/* testCatchPartiallyQualified */
95+
} catch (Level\Exception $e) {
96+
}
97+
98+
/* testNewRelative */
99+
$obj = new namespace\ClassName();
100+
101+
/* testNewFQN */
102+
$obj = new \Vendor\ClassName();
103+
104+
/* testNewUnqualified */
105+
$obj = new ClassName;
106+
107+
/* testNewPartiallyQualified */
108+
$obj = new Level\ClassName;
109+
110+
/* testDoubleColonRelative */
111+
$value = namespace\ClassName::property;
112+
113+
/* testDoubleColonFQN */
114+
$value = \ClassName::static_function();
115+
116+
/* testDoubleColonUnqualified */
117+
$value = ClassName::CONSTANT_NAME;
118+
119+
/* testDoubleColonPartiallyQualified */
120+
$value = Level\ClassName::CONSTANT_NAME['key'];
121+
122+
/* testInstanceOfRelative */
123+
$is = $obj instanceof namespace\ClassName;
124+
125+
/* testInstanceOfFQN */
126+
if ($obj instanceof \Full\ClassName) {}
127+
128+
/* testInstanceOfUnqualified */
129+
if ($a === $b && $obj instanceof ClassName && true) {}
130+
131+
/* testInstanceOfPartiallyQualified */
132+
$is = $obj instanceof Partially\ClassName;
133+
}
134+
}
135+
136+
/* testInvalidInPHP8Whitespace */
137+
namespace \ Sublevel
138+
\ function_name();
139+
140+
/* testInvalidInPHP8Comments */
141+
$value = \Fully
142+
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
143+
\Qualified
144+
/* comment */
145+
\Name
146+
// comment
147+
:: function_name();

0 commit comments

Comments
 (0)