Skip to content

Commit 63c2b22

Browse files
jrfnljrfnl
committed
PHP 8.0 | Tokenizer/PHP: tokenize the "|" for union types as T_TYPE_UNION
This adds a new block of logic to the `PHP::processAdditional()` method which changes the token code and type of `T_BITWISE_OR` `|` tokens in type declarations to `T_TYPE_UNION`. As the `PHP::processAdditional()` method walks backwards through the token stack, the arrow function backfill will not have been done yet, so for those some special conditions have been put in place. I've tried to limit the token walking within the new block as much as possible while still maintaining accuracy. This includes changing all union type operators in a single type declaration in one go, instead of on each individual `T_BITWISE_OR` token, which prevents the same logic having to be executed multiple times for multi-union types like `int|float|null`. Includes dedicated unit tests. Ref: https://wiki.php.net/rfc/union_types_v2
1 parent a0145ae commit 63c2b22

File tree

4 files changed

+384
-0
lines changed

4 files changed

+384
-0
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
126126
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
127127
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
128128
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
129+
<file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
130+
<file baseinstalldir="" name="BitwiseOrTest.php" role="test" />
129131
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
130132
<file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
131133
<file baseinstalldir="" name="ScopeSettingWithNamespaceOperatorTest.inc" role="test" />
@@ -2006,6 +2008,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20062008
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
20072009
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
20082010
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
2011+
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
2012+
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
20092013
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
20102014
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
20112015
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
@@ -2071,6 +2075,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20712075
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
20722076
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
20732077
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
2078+
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
2079+
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
20742080
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
20752081
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
20762082
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />

‎src/Tokenizers/PHP.php‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2176,6 +2176,177 @@ protected function processAdditional()
21762176
}
21772177
}
21782178

2179+
continue;
2180+
} else if ($this->tokens[$i]['code'] === T_BITWISE_OR) {
2181+
/*
2182+
Convert "|" to T_TYPE_UNION or leave as T_BITWISE_OR.
2183+
*/
2184+
2185+
$allowed = [
2186+
T_STRING => T_STRING,
2187+
T_CALLABLE => T_CALLABLE,
2188+
T_SELF => T_SELF,
2189+
T_PARENT => T_PARENT,
2190+
T_STATIC => T_STATIC,
2191+
T_FALSE => T_FALSE,
2192+
T_NULL => T_NULL,
2193+
T_NS_SEPARATOR => T_NS_SEPARATOR,
2194+
];
2195+
2196+
$suspectedType = null;
2197+
$typeTokenCount = 0;
2198+
2199+
for ($x = ($i + 1); $x < $numTokens; $x++) {
2200+
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
2201+
continue;
2202+
}
2203+
2204+
if (isset($allowed[$this->tokens[$x]['code']]) === true) {
2205+
++$typeTokenCount;
2206+
continue;
2207+
}
2208+
2209+
if ($typeTokenCount > 0
2210+
&& ($this->tokens[$x]['code'] === T_BITWISE_AND
2211+
|| $this->tokens[$x]['code'] === T_ELLIPSIS)
2212+
) {
2213+
// Skip past reference and variadic indicators for parameter types.
2214+
++$x;
2215+
continue;
2216+
}
2217+
2218+
if ($this->tokens[$x]['code'] === T_VARIABLE) {
2219+
// Parameter/Property defaults can not contain variables, so this could be a type.
2220+
$suspectedType = 'property or parameter';
2221+
break;
2222+
}
2223+
2224+
if ($this->tokens[$x]['code'] === T_DOUBLE_ARROW) {
2225+
// Possible arrow function.
2226+
$suspectedType = 'return';
2227+
break;
2228+
}
2229+
2230+
if ($this->tokens[$x]['code'] === T_SEMICOLON) {
2231+
// Possible abstract method or interface method.
2232+
$suspectedType = 'return';
2233+
break;
2234+
}
2235+
2236+
if ($this->tokens[$x]['code'] === T_OPEN_CURLY_BRACKET
2237+
&& isset($this->tokens[$x]['scope_condition']) === true
2238+
&& $this->tokens[$this->tokens[$x]['scope_condition']]['code'] === T_FUNCTION
2239+
) {
2240+
$suspectedType = 'return';
2241+
}
2242+
2243+
break;
2244+
}//end for
2245+
2246+
if ($typeTokenCount === 0 || isset($suspectedType) === false) {
2247+
// Definitely not a union type, move on.
2248+
continue;
2249+
}
2250+
2251+
$typeTokenCount = 0;
2252+
$unionOperators = [$i];
2253+
$confirmed = false;
2254+
2255+
for ($x = ($i - 1); $x >= 0; $x--) {
2256+
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
2257+
continue;
2258+
}
2259+
2260+
if (isset($allowed[$this->tokens[$x]['code']]) === true) {
2261+
++$typeTokenCount;
2262+
continue;
2263+
}
2264+
2265+
// Union types can't use the nullable operator, but be tolerant to parse errors.
2266+
if ($typeTokenCount > 0 && $this->tokens[$x]['code'] === T_NULLABLE) {
2267+
continue;
2268+
}
2269+
2270+
if ($this->tokens[$x]['code'] === T_BITWISE_OR) {
2271+
$unionOperators[] = $x;
2272+
continue;
2273+
}
2274+
2275+
if ($suspectedType === 'return' && $this->tokens[$x]['code'] === T_COLON) {
2276+
$confirmed = true;
2277+
break;
2278+
}
2279+
2280+
if ($suspectedType === 'property or parameter'
2281+
&& (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
2282+
|| $this->tokens[$x]['code'] === T_VAR)
2283+
) {
2284+
// This will also confirm constructor property promotion parameters, but that's fine.
2285+
$confirmed = true;
2286+
}
2287+
2288+
break;
2289+
}//end for
2290+
2291+
if ($confirmed === false
2292+
&& $suspectedType === 'property or parameter'
2293+
&& isset($this->tokens[$i]['nested_parenthesis']) === true
2294+
) {
2295+
$parens = $this->tokens[$i]['nested_parenthesis'];
2296+
$last = end($parens);
2297+
2298+
if (isset($this->tokens[$last]['parenthesis_owner']) === true
2299+
&& $this->tokens[$this->tokens[$last]['parenthesis_owner']]['code'] === T_FUNCTION
2300+
) {
2301+
$confirmed = true;
2302+
} else {
2303+
// No parenthesis owner set, this may be an arrow function which has not yet
2304+
// had additional processing done.
2305+
if (isset($this->tokens[$last]['parenthesis_opener']) === true) {
2306+
for ($x = ($this->tokens[$last]['parenthesis_opener'] - 1); $x >= 0; $x--) {
2307+
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
2308+
continue;
2309+
}
2310+
2311+
break;
2312+
}
2313+
2314+
if ($this->tokens[$x]['code'] === T_FN) {
2315+
for (--$x; $x >= 0; $x--) {
2316+
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true
2317+
|| $this->tokens[$x]['code'] === T_BITWISE_AND
2318+
) {
2319+
continue;
2320+
}
2321+
2322+
break;
2323+
}
2324+
2325+
if ($this->tokens[$x]['code'] !== T_FUNCTION) {
2326+
$confirmed = true;
2327+
}
2328+
}
2329+
}//end if
2330+
}//end if
2331+
2332+
unset($parens, $last);
2333+
}//end if
2334+
2335+
if ($confirmed === false) {
2336+
// Not a union type after all, move on.
2337+
continue;
2338+
}
2339+
2340+
foreach ($unionOperators as $x) {
2341+
$this->tokens[$x]['code'] = T_TYPE_UNION;
2342+
$this->tokens[$x]['type'] = 'T_TYPE_UNION';
2343+
2344+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
2345+
$line = $this->tokens[$x]['line'];
2346+
echo "\t* token $x on line $line changed from T_BITWISE_OR to T_TYPE_UNION".PHP_EOL;
2347+
}
2348+
}
2349+
21792350
continue;
21802351
} else if ($this->tokens[$i]['code'] === T_STATIC) {
21812352
for ($x = ($i - 1); $x > 0; $x--) {

‎tests/Core/Tokenizer/BitwiseOrTest.inc‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
<?php
2+
3+
/*
4+
* Type union or bitwise or.
5+
*/
6+
7+
/* testBitwiseOr1 */
8+
$result = $value | $test /* testBitwiseOr2 */ | $another;
9+
10+
class TypeUnion
11+
{
12+
/* testTypeUnionPropertySimple */
13+
public static Foo|Bar $obj;
14+
15+
/* testTypeUnionPropertyReverseModifierOrder */
16+
static protected int|float $number /* testBitwiseOrPropertyDefaultValue */ = E_WARNING | E_NOTICE;
17+
18+
private
19+
/* testTypeUnionPropertyMulti1 */
20+
array |
21+
/* testTypeUnionPropertyMulti2 */
22+
Traversable | // phpcs:ignore Stnd.Cat.Sniff
23+
false
24+
/* testTypeUnionPropertyMulti3 */
25+
| null $arrayOrFalse;
26+
27+
public function paramTypes(
28+
/* testTypeUnionParam1 */
29+
int|float $paramA /* testBitwiseOrParamDefaultValue */ = CONSTANT_A | CONSTANT_B,
30+
31+
/* testTypeUnionParam2 */
32+
Foo|\Bar /* testTypeUnionParam3 */ |Baz &...$paramB = null,
33+
) {
34+
/* testBitwiseOr3 */
35+
return (($a1 ^ $b1) |($a2 ^ $b2)) + $c;
36+
}
37+
38+
/* testTypeUnionReturnType */
39+
public function returnType() : int|false {}
40+
41+
/* testTypeUnionConstructorPropertyPromotion */
42+
public function __construct( public bool|null $property) {}
43+
44+
/* testTypeUnionAbstractMethodReturnType1 */
45+
abstract public function abstractMethod(): object|array /* testTypeUnionAbstractMethodReturnType2 */ |false;
46+
}
47+
48+
/* testTypeUnionClosureParamIllegalNullable */
49+
$closureWithParamType = function (?string|null $string) {};
50+
51+
/* testBitwiseOrClosureParamDefault */
52+
$closureWithReturnType = function ($string = NONSENSE | FAKE)/* testTypeUnionClosureReturn */ : \Package\MyA|PackageB {};
53+
54+
/* testTypeUnionArrowParam */
55+
$arrowWithParamType = fn (object|array $param, /* testBitwiseOrArrowParamDefault */ ?int $int = CONSTA | CONSTB )
56+
/* testBitwiseOrArrowExpression */
57+
=> $param | $int;
58+
59+
/* testTypeUnionArrowReturnType */
60+
$arrowWithReturnType = fn ($param) : int|null => $param * 10;
61+
62+
/* testBitwiseOrInArrayKey */
63+
$array = array(
64+
A | B => /* testBitwiseOrInArrayValue */ B | C
65+
);
66+
67+
/* testBitwiseOrInShortArrayKey */
68+
$array = [
69+
A | B => /* testBitwiseOrInShortArrayValue */ B | C
70+
];
71+
72+
/* testBitwiseOrTryCatch */
73+
try {
74+
} catch ( ExceptionA | ExceptionB $e ) {
75+
}
76+
77+
/* testBitwiseOrNonArrowFnFunctionCall */
78+
$obj->fn($something | $else);
79+
80+
/* testTypeUnionNonArrowFunctionDeclaration */
81+
function &fn(int|false $something) {}
82+
83+
/* testLiveCoding */
84+
// Intentional parse error. This has to be the last test in the file.
85+
return function( type|

0 commit comments

Comments
 (0)