Incorrect tokenization of GOTO statements in mixed PHP/HTML files

package.xml

@@ -157,6 +157,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
       <file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
       <file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
       <file baseinstalldir="" name="BitwiseOrTest.php" role="test" />
+      <file baseinstalldir="" name="GotoLabelTest.inc" role="test" />
+      <file baseinstalldir="" name="GotoLabelTest.php" role="test" />
       <file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
       <file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
       <file baseinstalldir="" name="ScopeSettingWithNamespaceOperatorTest.inc" role="test" />
@@ -2045,6 +2047,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
@@ -2117,6 +2121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />

src/Tokenizers/PHP.php

@@ -1572,6 +1572,7 @@ function return types. We want to keep the parenthesis map clean,
                 $stopTokens = [
                     T_CASE               => true,
                     T_SEMICOLON          => true,
+                    T_OPEN_TAG           => true,
                     T_OPEN_CURLY_BRACKET => true,
                     T_INLINE_THEN        => true,
                 ];

tests/Core/Tokenizer/GotoLabelTest.inc

@@ -0,0 +1,53 @@
+<?php
+
+/* testGotoStatement */
+goto marker;
+echo 'Foo';
+
+/* testGotoDeclaration */
+marker:
+echo 'Bar';
+
+/* testGotoStatementInLoop */
+for($i=0,$j=50; $i<100; $i++) {
+  while($j--) {
+    if($j==17) GOTO end;
+  }
+}
+echo "i = $i";
+?>
+<div><?php $cond ? TEST_A : TEST_B ?></div>
+
+<?php
+/* testGotoDeclarationOutsideLoop */
+end:
+echo 'j hit 17';
+
+switch ($x) {
+    /* testNotGotoDeclarationGlobalConstant */
+    case CONSTANT:
+        // Do something.
+        break;
+
+    /* testNotGotoDeclarationNamespacedConstant */
+    case MyNS\CONSTANT:
+        // Do something.
+        break;
+
+    /* testNotGotoDeclarationClassConstant */
+    case MyClass::CONSTANT:
+        // Do something.
+        break;
+
+    /* testNotGotoDeclarationClassProperty */
+    case $obj->property:
+        // Do something.
+        break;
+}
+
+switch (true) {
+    /* testNotGotoDeclarationGlobalConstantInTernary */
+    case $x === ($cond) ? CONST_A : CONST_B:
+        // Do something.
+        break;
+}

tests/Core/Tokenizer/GotoLabelTest.php

@@ -0,0 +1,171 @@
+<?php
+/**
+ * Tests the tokenization of goto declarations and statements.
+ *
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+
+class GotoLabelTest extends AbstractMethodUnitTest
+{
+
+
+    /**
+     * Verify that the label in a goto statement is tokenized as T_STRING.
+     *
+     * @param string $testMarker  The comment prefacing the target token.
+     * @param string $testContent The token content to expect.
+     *
+     * @dataProvider dataGotoStatement
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testGotoStatement($testMarker, $testContent)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $label = $this->getTargetToken($testMarker, T_STRING);
+
+        $this->assertInternalType('int', $label);
+        $this->assertSame($testContent, $tokens[$label]['content']);
+
+    }//end testGotoStatement()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testGotoStatement()
+     *
+     * @return array
+     */
+    public function dataGotoStatement()
+    {
+        return [
+            [
+                '/* testGotoStatement */',
+                'marker',
+            ],
+            [
+                '/* testGotoStatementInLoop */',
+                'end',
+            ],
+        ];
+
+    }//end dataGotoStatement()
+
+
+    /**
+     * Verify that the label in a goto declaration is tokenized as T_GOTO_LABEL.
+     *
+     * @param string $testMarker  The comment prefacing the target token.
+     * @param string $testContent The token content to expect.
+     *
+     * @dataProvider dataGotoDeclaration
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testGotoDeclaration($testMarker, $testContent)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $label = $this->getTargetToken($testMarker, T_GOTO_LABEL);
+
+        $this->assertInternalType('int', $label);
+        $this->assertSame($testContent, $tokens[$label]['content']);
+
+    }//end testGotoDeclaration()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testGotoDeclaration()
+     *
+     * @return array
+     */
+    public function dataGotoDeclaration()
+    {
+        return [
+            [
+                '/* testGotoDeclaration */',
+                'marker:',
+            ],
+            [
+                '/* testGotoDeclarationOutsideLoop */',
+                'end:',
+            ],
+        ];
+
+    }//end dataGotoDeclaration()
+
+
+    /**
+     * Verify that the constant used in a switch - case statement is not confused with a goto label.
+     *
+     * @param string $testMarker  The comment prefacing the target token.
+     * @param string $testContent The token content to expect.
+     *
+     * @dataProvider dataNotAGotoDeclaration
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testNotAGotoDeclaration($testMarker, $testContent)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+        $target = $this->getTargetToken($testMarker, [T_GOTO_LABEL, T_STRING], $testContent);
+
+        $this->assertSame(T_STRING, $tokens[$target]['code']);
+        $this->assertSame('T_STRING', $tokens[$target]['type']);
+
+    }//end testNotAGotoDeclaration()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testNotAGotoDeclaration()
+     *
+     * @return array
+     */
+    public function dataNotAGotoDeclaration()
+    {
+        return [
+            [
+                '/* testNotGotoDeclarationGlobalConstant */',
+                'CONSTANT',
+            ],
+            [
+                '/* testNotGotoDeclarationNamespacedConstant */',
+                'CONSTANT',
+            ],
+            [
+                '/* testNotGotoDeclarationClassConstant */',
+                'CONSTANT',
+            ],
+            [
+                '/* testNotGotoDeclarationClassProperty */',
+                'property',
+            ],
+            [
+                '/* testNotGotoDeclarationGlobalConstantInTernary */',
+                'CONST_A',
+            ],
+            [
+                '/* testNotGotoDeclarationGlobalConstantInTernary */',
+                'CONST_B',
+            ],
+        ];
+
+    }//end dataNotAGotoDeclaration()
+
+
+}//end class