diff --git a/src/wp-includes/html-api/class-wp-html-tag-processor.php b/src/wp-includes/html-api/class-wp-html-tag-processor.php index 17b3f400fcea6..5bb2d27986aa0 100644 --- a/src/wp-includes/html-api/class-wp-html-tag-processor.php +++ b/src/wp-includes/html-api/class-wp-html-tag-processor.php @@ -15,9 +15,6 @@ * - Prune the whitespace when removing classes/attributes: e.g. "a b c" -> "c" not " c". * This would increase the size of the changes for some operations but leave more * natural-looking output HTML. - * - Decode HTML character references within class names when matching. E.g. match having - * class `1<"2` needs to recognize `class="1<"2"`. Currently the Tag Processor - * will fail to find the right tag if the class name is encoded as such. * - Properly decode HTML character references in `get_attribute()`. PHP's * `html_entity_decode()` is wrong in a couple ways: it doesn't account for the * no-ambiguous-ampersand rule, and it improperly handles the way semicolons may @@ -107,6 +104,56 @@ * given, it will return `true` (the only way to set `false` for an * attribute is to remove it). * + * #### When matching fails + * + * When `next_tag()` returns `false` it could mean different things: + * + * - The requested tag wasn't found in the input document. + * - The input document ended in the middle of an HTML syntax element. + * + * When a document ends in the middle of a syntax element it will pause + * the processor. This is to make it possible in the future to extend the + * input document and proceed - an important requirement for chunked + * streaming parsing of a document. + * + * Example: + * + * $processor = new WP_HTML_Tag_Processor( 'This
` inside an HTML comment. + * - STYLE content is raw text. + * - TITLE content is plain text but character references are decoded. + * - TEXTAREA content is plain text but character references are decoded. + * - XMP (deprecated) content is raw text. + * * ### Modifying HTML attributes for a found tag * * Once you've found the start of an opening tag you can modify @@ -241,9 +288,13 @@ * double-quoted strings, meaning that attributes on input with single-quoted or * unquoted values will appear in the output with double-quotes. * + * scripts aren't processed + * * @since 6.2.0 * @since 6.2.1 Fix: Support for various invalid comments; attribute updates are case-insensitive. * @since 6.3.2 Fix: Skip HTML-like content inside rawtext elements such as STYLE. + * @since 6.5.0 Pauses processor when input ends in an incomplete syntax token. + * Introduces "special" elements which act like void elements, e.g. STYLE. */ class WP_HTML_Tag_Processor { /** @@ -316,6 +367,20 @@ class WP_HTML_Tag_Processor { */ private $stop_on_tag_closers; + /** + * What the parser is up to. + * + * @since {WP_VERSION} + * + * @see WP_HTML_Tag_Processor::STATE_READY + * @see WP_HTML_Tag_Processor::STATE_COMPLETE + * @see WP_HTML_Tag_Processor::STATE_INCOMPLETE + * @see WP_HTML_Tag_Processor::STATE_MATCHED_TAG + * + * @var string + */ + private $parser_state = self::STATE_READY; + /** * How many bytes from the original HTML document have been read and parsed. * @@ -544,6 +609,7 @@ public function __construct( $html ) { * Finds the next tag matching the $query. * * @since 6.2.0 + * @since 6.5.0 No longer processes incomplete tokens at end of document as text; pauses the processor instead. * * @param array|string|null $query { * Optional. Which tag name to find, having which class, etc. Default is to find any tag. @@ -562,86 +628,129 @@ public function next_tag( $query = null ) { $already_found = 0; do { - if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + if ( false === $this->next_token() ) { return false; } - // Find the next tag if it exists. - if ( false === $this->parse_next_tag() ) { - $this->bytes_already_parsed = strlen( $this->html ); - - return false; - } - - // Parse all of its attributes. - while ( $this->parse_next_attribute() ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { continue; } - // Ensure that the tag closes before the end of the document. - if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { - return false; + if ( $this->matches() ) { + ++$already_found; } + } while ( $already_found < $this->sought_match_offset ); - $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed ); - if ( false === $tag_ends_at ) { - return false; - } - $this->token_length = $tag_ends_at - $this->token_starts_at; - $this->bytes_already_parsed = $tag_ends_at; + return true; + } - // Finally, check if the parsed tag and its attributes match the search query. - if ( $this->matches() ) { - ++$already_found; + public function next_token() { + $this->get_updated_html(); + $was_at = $this->bytes_already_parsed; + + // Don't proceed if there's nothing more to scan. + if ( + self::STATE_COMPLETE === $this->parser_state || + self::STATE_INCOMPLETE === $this->parser_state + ) { + return false; + } + + /* + * The next step in the parsing loop determines the parsing state; + * clear it so that state doesn't linger from the previous step. + */ + $this->parser_state = self::STATE_READY; + + if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_COMPLETE; + return false; + } + + // Find the next tag if it exists. + if ( false === $this->parse_next_tag() ) { + if ( self::STATE_INCOMPLETE === $this->parser_state ) { + $this->bytes_already_parsed = $was_at; } - /* - * For non-DATA sections which might contain text that looks like HTML tags but - * isn't, scan with the appropriate alternative mode. Looking at the first letter - * of the tag name as a pre-check avoids a string allocation when it's not needed. - */ - $t = $this->html[ $this->tag_name_starts_at ]; - if ( - ! $this->is_closing_tag && + return false; + } + + // Parse all of its attributes. + while ( $this->parse_next_attribute() ) { + continue; + } + + // Ensure that the tag closes before the end of the document. + if ( + self::STATE_INCOMPLETE === $this->parser_state || + $this->bytes_already_parsed >= strlen( $this->html ) + ) { + // Does this appropriately clear state (parsed attributes)? + $this->parser_state = self::STATE_INCOMPLETE; + $this->bytes_already_parsed = $was_at; + + return false; + } + + $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed ); + if ( false === $tag_ends_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + $this->bytes_already_parsed = $was_at; + + return false; + } + $this->parser_state = self::STATE_MATCHED_TAG; + $this->token_length = $tag_ends_at - $this->token_starts_at; + $this->bytes_already_parsed = $tag_ends_at; + + /* + * For non-DATA sections which might contain text that looks like HTML tags but + * isn't, scan with the appropriate alternative mode. Looking at the first letter + * of the tag name as a pre-check avoids a string allocation when it's not needed. + */ + $t = $this->html[ $this->tag_name_starts_at ]; + if ( + ! $this->is_closing_tag && + ( + 'i' === $t || 'I' === $t || + 'n' === $t || 'N' === $t || + 's' === $t || 'S' === $t || + 't' === $t || 'T' === $t || + 'x' === $t || 'X' === $t + ) + ) { + $tag_name = $this->get_tag(); + + if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) { + $this->parser_state = self::STATE_INCOMPLETE; + $this->bytes_already_parsed = $was_at; + + return false; + } elseif ( + ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) && + ! $this->skip_rcdata( $tag_name ) + ) { + $this->parser_state = self::STATE_INCOMPLETE; + $this->bytes_already_parsed = $was_at; + + return false; + } elseif ( ( - 'i' === $t || 'I' === $t || - 'n' === $t || 'N' === $t || - 's' === $t || 'S' === $t || - 't' === $t || 'T' === $t - ) ) { - $tag_name = $this->get_tag(); - - if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) { - $this->bytes_already_parsed = strlen( $this->html ); - return false; - } elseif ( - ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) && - ! $this->skip_rcdata( $tag_name ) - ) { - $this->bytes_already_parsed = strlen( $this->html ); - return false; - } elseif ( - ( - 'IFRAME' === $tag_name || - 'NOEMBED' === $tag_name || - 'NOFRAMES' === $tag_name || - 'NOSCRIPT' === $tag_name || - 'STYLE' === $tag_name - ) && - ! $this->skip_rawtext( $tag_name ) - ) { - /* - * "XMP" should be here too but its rules are more complicated and require the - * complexity of the HTML Processor (it needs to close out any open P element, - * meaning it can't be skipped here or else the HTML Processor will lose its - * place). For now, it can be ignored as it's a rare HTML tag in practice and - * any normative HTML should be using PRE instead. - */ - $this->bytes_already_parsed = strlen( $this->html ); - return false; - } + 'IFRAME' === $tag_name || + 'NOEMBED' === $tag_name || + 'NOFRAMES' === $tag_name || + 'STYLE' === $tag_name || + 'XMP' === $tag_name + ) && + ! $this->skip_rawtext( $tag_name ) + ) { + $this->parser_state = self::STATE_INCOMPLETE; + $this->bytes_already_parsed = $was_at; + + return false; } - } while ( $already_found < $this->sought_match_offset ); + } return true; } @@ -664,6 +773,10 @@ public function next_tag( $query = null ) { * @since 6.4.0 */ public function class_list() { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { + return; + } + /** @var string $class contains the string value of the class attribute, with character references decoded. */ $class = $this->get_attribute( 'class' ); @@ -719,7 +832,7 @@ public function class_list() { * @return bool|null Whether the matched tag contains the given class name, or null if not matched. */ public function has_class( $wanted_class ) { - if ( ! $this->tag_name_starts_at ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { return null; } @@ -816,7 +929,8 @@ public function has_class( $wanted_class ) { * @return bool Whether the bookmark was successfully created. */ public function set_bookmark( $name ) { - if ( null === $this->tag_name_starts_at ) { + // It only makes sense to set a bookmark if the parser has paused on a concrete token. + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { return false; } @@ -895,7 +1009,6 @@ private function skip_rcdata( $tag_name ) { // Fail if there is no possible tag closer. if ( false === $at || ( $at + $tag_length ) >= $doc_length ) { - $this->bytes_already_parsed = $doc_length; return false; } @@ -923,6 +1036,10 @@ private function skip_rcdata( $tag_name ) { $at += $tag_length; $this->bytes_already_parsed = $at; + if ( $at >= strlen( $html ) ) { + return false; + } + /* * Ensure that the tag name terminates to avoid matching on * substrings of a longer tag name. For example, the sequence @@ -1107,6 +1224,10 @@ private function parse_next_tag() { while ( false !== $at && $at < $doc_length ) { $at = strpos( $html, '<', $at ); + /* + * This does not imply an incomplete parse; it indicates that there + * can be nothing left in the document other than a #text node. + */ if ( false === $at ) { return false; } @@ -1148,6 +1269,8 @@ private function parse_next_tag() { * the document. There is nothing left to parse. */ if ( $at + 1 >= strlen( $html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1168,6 +1291,8 @@ private function parse_next_tag() { $closer_at = $at + 4; // If it's not possible to close the comment then there is nothing more to scan. if ( strlen( $html ) <= $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1188,6 +1313,8 @@ private function parse_next_tag() { while ( ++$closer_at < strlen( $html ) ) { $closer_at = strpos( $html, '--', $closer_at ); if ( false === $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1220,6 +1347,8 @@ private function parse_next_tag() { ) { $closer_at = strpos( $html, ']]>', $at + 9 ); if ( false === $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1244,6 +1373,8 @@ private function parse_next_tag() { ) { $closer_at = strpos( $html, '>', $at + 9 ); if ( false === $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1276,6 +1407,8 @@ private function parse_next_tag() { if ( '?' === $html[ $at + 1 ] ) { $closer_at = strpos( $html, '>', $at + 2 ); if ( false === $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1292,6 +1425,8 @@ private function parse_next_tag() { if ( $this->is_closing_tag ) { $closer_at = strpos( $html, '>', $at + 3 ); if ( false === $closer_at ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1316,6 +1451,8 @@ private function parse_next_attribute() { // Skip whitespace and slashes. $this->bytes_already_parsed += strspn( $this->html, " \t\f\r\n/", $this->bytes_already_parsed ); if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1338,11 +1475,15 @@ private function parse_next_attribute() { $attribute_name = substr( $this->html, $attribute_start, $name_length ); $this->bytes_already_parsed += $name_length; if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } $this->skip_whitespace(); if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1351,6 +1492,8 @@ private function parse_next_attribute() { ++$this->bytes_already_parsed; $this->skip_whitespace(); if ( $this->bytes_already_parsed >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1377,6 +1520,8 @@ private function parse_next_attribute() { } if ( $attribute_end >= strlen( $this->html ) ) { + $this->parser_state = self::STATE_INCOMPLETE; + return false; } @@ -1443,7 +1588,6 @@ private function skip_whitespace() { * @since 6.2.0 */ private function after_tag() { - $this->get_updated_html(); $this->token_starts_at = null; $this->token_length = null; $this->tag_name_starts_at = null; @@ -1786,6 +1930,10 @@ private static function sort_start_ascending( $a, $b ) { * @return string|boolean|null Value of enqueued update if present, otherwise false. */ private function get_enqueued_attribute_value( $comparable_name ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { + return false; + } + if ( ! isset( $this->lexical_updates[ $comparable_name ] ) ) { return false; } @@ -1853,7 +2001,7 @@ private function get_enqueued_attribute_value( $comparable_name ) { * @return string|true|null Value of attribute or `null` if not available. Boolean attributes return `true`. */ public function get_attribute( $name ) { - if ( null === $this->tag_name_starts_at ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { return null; } @@ -1933,7 +2081,10 @@ public function get_attribute( $name ) { * @return array|null List of attribute names, or `null` when no tag opener is matched. */ public function get_attribute_names_with_prefix( $prefix ) { - if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) { + if ( + self::STATE_MATCHED_TAG !== $this->parser_state || + $this->is_closing_tag + ) { return null; } @@ -1965,7 +2116,7 @@ public function get_attribute_names_with_prefix( $prefix ) { * @return string|null Name of currently matched tag in input HTML, or `null` if none found. */ public function get_tag() { - if ( null === $this->tag_name_starts_at ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { return null; } @@ -1992,7 +2143,7 @@ public function get_tag() { * @return bool Whether the currently matched tag contains the self-closing flag. */ public function has_self_closing_flag() { - if ( ! $this->tag_name_starts_at ) { + if ( self::STATE_MATCHED_TAG !== $this->parser_state ) { return false; } @@ -2024,7 +2175,10 @@ public function has_self_closing_flag() { * @return bool Whether the current tag is a tag closer. */ public function is_tag_closer() { - return $this->is_closing_tag; + return ( + self::STATE_MATCHED_TAG === $this->parser_state && + $this->is_closing_tag + ); } /** @@ -2044,7 +2198,10 @@ public function is_tag_closer() { * @return bool Whether an attribute value was set. */ public function set_attribute( $name, $value ) { - if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) { + if ( + self::STATE_MATCHED_TAG !== $this->parser_state || + $this->is_closing_tag + ) { return false; } @@ -2177,7 +2334,10 @@ public function set_attribute( $name, $value ) { * @return bool Whether an attribute was removed. */ public function remove_attribute( $name ) { - if ( $this->is_closing_tag ) { + if ( + self::STATE_MATCHED_TAG !== $this->parser_state || + $this->is_closing_tag + ) { return false; } @@ -2254,13 +2414,14 @@ public function remove_attribute( $name ) { * @return bool Whether the class was set to be added. */ public function add_class( $class_name ) { - if ( $this->is_closing_tag ) { + if ( + self::STATE_MATCHED_TAG !== $this->parser_state || + $this->is_closing_tag + ) { return false; } - if ( null !== $this->tag_name_starts_at ) { - $this->classname_updates[ $class_name ] = self::ADD_CLASS; - } + $this->classname_updates[ $class_name ] = self::ADD_CLASS; return true; } @@ -2274,7 +2435,10 @@ public function add_class( $class_name ) { * @return bool Whether the class was set to be removed. */ public function remove_class( $class_name ) { - if ( $this->is_closing_tag ) { + if ( + self::STATE_MATCHED_TAG !== $this->parser_state || + $this->is_closing_tag + ) { return false; } @@ -2480,4 +2644,9 @@ private function matches() { return true; } + + const STATE_READY = 'READY: The parser is waiting for a state transition; it may not have started, or it may have been interrupted, or it may be waiting to restart after pausing.'; + const STATE_COMPLETE = 'COMPLETE: The parser has reached the end of the document without truncating any possible tokens. There is nothing left to scan.'; + const STATE_INCOMPLETE = 'INCOMPLETE: The parser has reached the end of the document but it appears as thought the HTML is truncated inside a token. It has backed up to the last-known complete state and will not continue parsing.'; + const STATE_MATCHED_TAG = 'MATCHED_TAG: The parser has found a tag and paused to allow reading from and modifying its attributes.'; } diff --git a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php index 4469f90c4f276..f500512deeacd 100644 --- a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php +++ b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php @@ -1758,9 +1758,8 @@ public function test_setting_a_boolean_attribute_to_a_string_value_adds_explicit * @covers WP_HTML_Tag_Processor::next_tag */ public function test_unclosed_script_tag_should_not_cause_an_infinite_loop() { - $p = new WP_HTML_Tag_Processor( '