PHP 8.0 | Tokenizer/PHP: arrow function backfill vs PHP8 union types

As the `PHP::processAdditional()` method walks backwards through the file, by the time the `fn` keyword backfill logic is hit, any union type `|` tokens will have already been converted to `T_TYPE_UNION`.

So, to make the arrow function backfill compatible with PHP 8 union types, the `T_TYPE_UNION` token needs to be added to the "allowed tokens" (`$ignore`) array.

Includes unit tests.

Author: jrfnl

src/Tokenizers/PHP.php

@@ -1999,6 +1999,7 @@ protected function processAdditional()
                         T_PARENT       => T_PARENT,
                         T_SELF         => T_SELF,
                         T_STATIC       => T_STATIC,
+                        T_TYPE_UNION   => T_TYPE_UNION,
                     ];
 
                     $closer = $this->tokens[$x]['parenthesis_closer'];

tests/Core/Tokenizer/BackfillFnTokenTest.inc

@@ -81,6 +81,12 @@ fn(array $a) : array => $a;
 /* testStaticReturnType */
 fn(array $a) : static => $a;
 
+/* testUnionParamType */
+$arrowWithUnionParam = fn(int|float $param) : SomeClass => new SomeClass($param);
+
+/* testUnionReturnType */
+$arrowWithUnionReturn = fn($param) : int|float => $param | 10;
+
 /* testTernary */
 $fn = fn($a) => $a ? /* testTernaryThen */ fn() : string => 'a' : /* testTernaryElse */ fn() : string => 'b';
 
@@ -130,6 +136,9 @@ $a = MyNS\Sub\Fn($param);
 /* testNonArrowNamespaceOperatorFunctionCall */
 $a = namespace\fn($param);
 
+/* testNonArrowFunctionNameWithUnionTypes */
+function fn(int|float $param) : string|null {}
+
 /* testLiveCoding */
 // Intentional parse error. This has to be the last test in the file.
 $fn = fn

tests/Core/Tokenizer/BackfillFnTokenTest.php

@@ -531,6 +531,62 @@ public function testKeywordReturnTypes()
     }//end testKeywordReturnTypes()
 
 
+    /**
+     * Test arrow function with a union parameter type.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
+     *
+     * @return void
+     */
+    public function testUnionParamType()
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $token = $this->getTargetToken('/* testUnionParamType */', T_FN);
+        $this->backfillHelper($token);
+
+        $this->assertSame($tokens[$token]['scope_opener'], ($token + 13), 'Scope opener is not the arrow token');
+        $this->assertSame($tokens[$token]['scope_closer'], ($token + 21), 'Scope closer is not the semicolon token');
+
+        $opener = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$opener]['scope_opener'], ($token + 13), 'Opener scope opener is not the arrow token');
+        $this->assertSame($tokens[$opener]['scope_closer'], ($token + 21), 'Opener scope closer is not the semicolon token');
+
+        $closer = $tokens[$token]['scope_closer'];
+        $this->assertSame($tokens[$closer]['scope_opener'], ($token + 13), 'Closer scope opener is not the arrow token');
+        $this->assertSame($tokens[$closer]['scope_closer'], ($token + 21), 'Closer scope closer is not the semicolon token');
+
+    }//end testUnionParamType()
+
+
+    /**
+     * Test arrow function with a union return type.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
+     *
+     * @return void
+     */
+    public function testUnionReturnType()
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $token = $this->getTargetToken('/* testUnionReturnType */', T_FN);
+        $this->backfillHelper($token);
+
+        $this->assertSame($tokens[$token]['scope_opener'], ($token + 11), 'Scope opener is not the arrow token');
+        $this->assertSame($tokens[$token]['scope_closer'], ($token + 18), 'Scope closer is not the semicolon token');
+
+        $opener = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$opener]['scope_opener'], ($token + 11), 'Opener scope opener is not the arrow token');
+        $this->assertSame($tokens[$opener]['scope_closer'], ($token + 18), 'Opener scope closer is not the semicolon token');
+
+        $closer = $tokens[$token]['scope_closer'];
+        $this->assertSame($tokens[$closer]['scope_opener'], ($token + 11), 'Closer scope opener is not the arrow token');
+        $this->assertSame($tokens[$closer]['scope_closer'], ($token + 18), 'Closer scope closer is not the semicolon token');
+
+    }//end testUnionReturnType()
+
+
     /**
      * Test arrow functions used in ternary operators.
      *
@@ -690,6 +746,7 @@ public function dataNotAnArrowFunction()
                 'Fn',
             ],
             ['/* testNonArrowNamespaceOperatorFunctionCall */'],
+            ['/* testNonArrowFunctionNameWithUnionTypes */'],
             ['/* testLiveCoding */'],
         ];