Skip to content

Commit ba63323

Browse files
jrfnljrfnl
committed
PHP 8.0: "undo" namespaced names as single token
As per the proposal in 3041. This effectively "undoes" the new PHP 8.0 tokenization of identifier names for PHPCS 3.x. Includes extensive unit tests to ensure the correct re-tokenization as well as that the rest of the tokenization is not adversely affected by this change. Includes preventing `function ...` within a group use statement from breaking the retokenization. Includes fixing the nullable tokenization when combined with any of the new PHP 8 identifier name tokens.
1 parent 1371c59 commit ba63323

File tree

5 files changed

+1562
-9
lines changed

5 files changed

+1562
-9
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
121121
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
122122
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
123123
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
124+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
125+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
124126
</dir>
125127
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
126128
<file baseinstalldir="" name="AllTests.php" role="test" />
@@ -1997,6 +1999,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
19971999
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
19982000
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
19992001
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2002+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2003+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20002004
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20012005
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20022006
</filelist>
@@ -2058,6 +2062,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20582062
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
20592063
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
20602064
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2065+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2066+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20612067
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20622068
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20632069
<ignore name="bin/phpcs.bat" />

‎src/Tokenizers/PHP.php‎

Lines changed: 100 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -815,6 +815,81 @@ protected function tokenize($string)
815815
continue;
816816
}//end if
817817

818+
/*
819+
As of PHP 8.0 fully qualified, partially qualified and namespace relative
820+
identifier names are tokenized differently.
821+
This "undoes" the new tokenization so the tokenization will be the same in
822+
in PHP 5, 7 and 8.
823+
*/
824+
825+
if (PHP_VERSION_ID >= 80000
826+
&& $tokenIsArray === true
827+
&& ($token[0] === T_NAME_QUALIFIED
828+
|| $token[0] === T_NAME_FULLY_QUALIFIED
829+
|| $token[0] === T_NAME_RELATIVE)
830+
) {
831+
$name = $token[1];
832+
833+
if ($token[0] === T_NAME_FULLY_QUALIFIED) {
834+
$newToken = [];
835+
$newToken['code'] = T_NS_SEPARATOR;
836+
$newToken['type'] = 'T_NS_SEPARATOR';
837+
$newToken['content'] = '\\';
838+
$finalTokens[$newStackPtr] = $newToken;
839+
++$newStackPtr;
840+
841+
$name = ltrim($name, '\\');
842+
}
843+
844+
if ($token[0] === T_NAME_RELATIVE) {
845+
$newToken = [];
846+
$newToken['code'] = T_NAMESPACE;
847+
$newToken['type'] = 'T_NAMESPACE';
848+
$newToken['content'] = substr($name, 0, 9);
849+
$finalTokens[$newStackPtr] = $newToken;
850+
++$newStackPtr;
851+
852+
$newToken = [];
853+
$newToken['code'] = T_NS_SEPARATOR;
854+
$newToken['type'] = 'T_NS_SEPARATOR';
855+
$newToken['content'] = '\\';
856+
$finalTokens[$newStackPtr] = $newToken;
857+
++$newStackPtr;
858+
859+
$name = substr($name, 10);
860+
}
861+
862+
$parts = explode('\\', $name);
863+
$partCount = count($parts);
864+
$lastPart = ($partCount - 1);
865+
866+
foreach ($parts as $i => $part) {
867+
$newToken = [];
868+
$newToken['code'] = T_STRING;
869+
$newToken['type'] = 'T_STRING';
870+
$newToken['content'] = $part;
871+
$finalTokens[$newStackPtr] = $newToken;
872+
++$newStackPtr;
873+
874+
if ($i !== $lastPart) {
875+
$newToken = [];
876+
$newToken['code'] = T_NS_SEPARATOR;
877+
$newToken['type'] = 'T_NS_SEPARATOR';
878+
$newToken['content'] = '\\';
879+
$finalTokens[$newStackPtr] = $newToken;
880+
++$newStackPtr;
881+
}
882+
}
883+
884+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
885+
$type = Util\Tokens::tokenName($token[0]);
886+
$content = Util\Common::prepareForOutput($token[1]);
887+
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
888+
}
889+
890+
continue;
891+
}//end if
892+
818893
/*
819894
Before PHP 7.0, the "yield from" was tokenized as
820895
T_YIELD, T_WHITESPACE and T_STRING. So look for
@@ -1131,7 +1206,7 @@ protected function tokenize($string)
11311206
* Check if the next non-empty token is one of the tokens which can be used
11321207
* in type declarations. If not, it's definitely a ternary.
11331208
* At this point, the only token types which need to be taken into consideration
1134-
* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
1209+
* as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
11351210
*/
11361211

11371212
$lastRelevantNonEmpty = null;
@@ -1148,6 +1223,9 @@ protected function tokenize($string)
11481223
}
11491224

11501225
if ($tokenType === T_STRING
1226+
|| $tokenType === T_NAME_FULLY_QUALIFIED
1227+
|| $tokenType === T_NAME_RELATIVE
1228+
|| $tokenType === T_NAME_QUALIFIED
11511229
|| $tokenType === T_ARRAY
11521230
|| $tokenType === T_NS_SEPARATOR
11531231
) {
@@ -1159,7 +1237,10 @@ protected function tokenize($string)
11591237
&& isset($lastRelevantNonEmpty) === false)
11601238
|| ($lastRelevantNonEmpty === T_ARRAY
11611239
&& $tokenType === '(')
1162-
|| ($lastRelevantNonEmpty === T_STRING
1240+
|| (($lastRelevantNonEmpty === T_STRING
1241+
|| $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
1242+
|| $lastRelevantNonEmpty === T_NAME_RELATIVE
1243+
|| $lastRelevantNonEmpty === T_NAME_QUALIFIED)
11631244
&& ($tokenType === T_DOUBLE_COLON
11641245
|| $tokenType === '('
11651246
|| $tokenType === ':'))
@@ -1304,6 +1385,10 @@ protected function tokenize($string)
13041385
tokenized as T_STRING even if it appears to be a different token,
13051386
such as when writing code like: function default(): foo
13061387
so go forward and change the token type before it is processed.
1388+
1389+
Note: this should not be done for `function Level\Name` within a
1390+
group use statement for the PHP 8 identifier name tokens as it
1391+
would interfere with the re-tokenization of those.
13071392
*/
13081393

13091394
if ($tokenIsArray === true
@@ -1321,7 +1406,10 @@ protected function tokenize($string)
13211406
}
13221407
}
13231408

1324-
if ($x < $numTokens && is_array($tokens[$x]) === true) {
1409+
if ($x < $numTokens
1410+
&& is_array($tokens[$x]) === true
1411+
&& $tokens[$x][0] !== T_NAME_QUALIFIED
1412+
) {
13251413
if (PHP_CODESNIFFER_VERBOSITY > 1) {
13261414
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
13271415
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
@@ -1377,12 +1465,15 @@ function return types. We want to keep the parenthesis map clean,
13771465
&& $tokens[$x] === ':'
13781466
) {
13791467
$allowed = [
1380-
T_STRING => T_STRING,
1381-
T_ARRAY => T_ARRAY,
1382-
T_CALLABLE => T_CALLABLE,
1383-
T_SELF => T_SELF,
1384-
T_PARENT => T_PARENT,
1385-
T_NS_SEPARATOR => T_NS_SEPARATOR,
1468+
T_STRING => T_STRING,
1469+
T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
1470+
T_NAME_RELATIVE => T_NAME_RELATIVE,
1471+
T_NAME_QUALIFIED => T_NAME_QUALIFIED,
1472+
T_ARRAY => T_ARRAY,
1473+
T_CALLABLE => T_CALLABLE,
1474+
T_SELF => T_SELF,
1475+
T_PARENT => T_PARENT,
1476+
T_NS_SEPARATOR => T_NS_SEPARATOR,
13861477
];
13871478

13881479
$allowed += Util\Tokens::$emptyTokens;

‎src/Util/Tokens.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,18 @@
129129
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
130130
}
131131

132+
if (defined('T_NAME_QUALIFIED') === false) {
133+
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
134+
}
135+
136+
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
137+
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
138+
}
139+
140+
if (defined('T_NAME_RELATIVE') === false) {
141+
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
142+
}
143+
132144
// Tokens used for parsing doc blocks.
133145
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
134146
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
<?php
2+
3+
/* testNamespaceDeclaration */
4+
namespace Package;
5+
6+
/* testNamespaceDeclarationWithLevels */
7+
namespace Vendor\SubLevel\Domain;
8+
9+
/* testUseStatement */
10+
use ClassName;
11+
12+
/* testUseStatementWithLevels */
13+
use Vendor\Level\Domain;
14+
15+
/* testFunctionUseStatement */
16+
use function function_name;
17+
18+
/* testFunctionUseStatementWithLevels */
19+
use function Vendor\Level\function_in_ns;
20+
21+
/* testConstantUseStatement */
22+
use const CONSTANT_NAME;
23+
24+
/* testConstantUseStatementWithLevels */
25+
use const Vendor\Level\OTHER_CONSTANT;
26+
27+
/* testMultiUseUnqualified */
28+
use UnqualifiedClassName,
29+
/* testMultiUsePartiallyQualified */
30+
Sublevel\PartiallyClassName;
31+
32+
/* testGroupUseStatement */
33+
use Vendor\Level\{
34+
AnotherDomain,
35+
function function_grouped,
36+
const CONSTANT_GROUPED,
37+
Sub\YetAnotherDomain,
38+
function SubLevelA\function_grouped_too,
39+
const SubLevelB\CONSTANT_GROUPED_TOO,
40+
};
41+
42+
/* testClassName */
43+
class MyClass
44+
/* testExtendedFQN */
45+
extends \Vendor\Level\FQN
46+
/* testImplementsRelative */
47+
implements namespace\Name,
48+
/* testImplementsFQN */
49+
\Fully\Qualified,
50+
/* testImplementsUnqualified */
51+
Unqualified,
52+
/* testImplementsPartiallyQualified */
53+
Sub\Level\Name
54+
{
55+
/* testFunctionName */
56+
public function function_name(
57+
/* testTypeDeclarationRelative */
58+
?namespace\Name|object $paramA,
59+
60+
/* testTypeDeclarationFQN */
61+
\Fully\Qualified\Name $paramB,
62+
63+
/* testTypeDeclarationUnqualified */
64+
Unqualified|false $paramC,
65+
66+
/* testTypeDeclarationPartiallyQualified */
67+
?Sublevel\Name $paramD,
68+
69+
/* testReturnTypeFQN */
70+
) : ?\Name {
71+
72+
try {
73+
/* testFunctionCallRelative */
74+
echo NameSpace\function_name();
75+
76+
/* testFunctionCallFQN */
77+
echo \Vendor\Package\function_name();
78+
79+
/* testFunctionCallUnqualified */
80+
echo function_name();
81+
82+
/* testFunctionPartiallyQualified */
83+
echo Level\function_name();
84+
85+
/* testCatchRelative */
86+
} catch (namespace\SubLevel\Exception $e) {
87+
88+
/* testCatchFQN */
89+
} catch (\Exception $e) {
90+
91+
/* testCatchUnqualified */
92+
} catch (Exception $e) {
93+
94+
/* testCatchPartiallyQualified */
95+
} catch (Level\Exception $e) {
96+
}
97+
98+
/* testNewRelative */
99+
$obj = new namespace\ClassName();
100+
101+
/* testNewFQN */
102+
$obj = new \Vendor\ClassName();
103+
104+
/* testNewUnqualified */
105+
$obj = new ClassName;
106+
107+
/* testNewPartiallyQualified */
108+
$obj = new Level\ClassName;
109+
110+
/* testDoubleColonRelative */
111+
$value = namespace\ClassName::property;
112+
113+
/* testDoubleColonFQN */
114+
$value = \ClassName::static_function();
115+
116+
/* testDoubleColonUnqualified */
117+
$value = ClassName::CONSTANT_NAME;
118+
119+
/* testDoubleColonPartiallyQualified */
120+
$value = Level\ClassName::CONSTANT_NAME['key'];
121+
122+
/* testInstanceOfRelative */
123+
$is = $obj instanceof namespace\ClassName;
124+
125+
/* testInstanceOfFQN */
126+
if ($obj instanceof \Full\ClassName) {}
127+
128+
/* testInstanceOfUnqualified */
129+
if ($a === $b && $obj instanceof ClassName && true) {}
130+
131+
/* testInstanceOfPartiallyQualified */
132+
$is = $obj instanceof Partially\ClassName;
133+
}
134+
}
135+
136+
/* testInvalidInPHP8Whitespace */
137+
namespace \ Sublevel
138+
\ function_name();
139+
140+
/* testInvalidInPHP8Comments */
141+
$value = \Fully
142+
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
143+
\Qualified
144+
/* comment */
145+
\Name
146+
// comment
147+
:: function_name();

0 commit comments

Comments
 (0)