diff --git a/src/wp-includes/html-api/class-wp-html-tag-processor.php b/src/wp-includes/html-api/class-wp-html-tag-processor.php
index 17b3f400fcea6..5bb2d27986aa0 100644
--- a/src/wp-includes/html-api/class-wp-html-tag-processor.php
+++ b/src/wp-includes/html-api/class-wp-html-tag-processor.php
@@ -15,9 +15,6 @@
* - Prune the whitespace when removing classes/attributes: e.g. "a b c" -> "c" not " c".
* This would increase the size of the changes for some operations but leave more
* natural-looking output HTML.
- * - Decode HTML character references within class names when matching. E.g. match having
- * class `1<"2` needs to recognize `class="1<"2"`. Currently the Tag Processor
- * will fail to find the right tag if the class name is encoded as such.
* - Properly decode HTML character references in `get_attribute()`. PHP's
* `html_entity_decode()` is wrong in a couple ways: it doesn't account for the
* no-ambiguous-ampersand rule, and it improperly handles the way semicolons may
@@ -107,6 +104,56 @@
* given, it will return `true` (the only way to set `false` for an
* attribute is to remove it).
*
+ * #### When matching fails
+ *
+ * When `next_tag()` returns `false` it could mean different things:
+ *
+ * - The requested tag wasn't found in the input document.
+ * - The input document ended in the middle of an HTML syntax element.
+ *
+ * When a document ends in the middle of a syntax element it will pause
+ * the processor. This is to make it possible in the future to extend the
+ * input document and proceed - an important requirement for chunked
+ * streaming parsing of a document.
+ *
+ * Example:
+ *
+ * $processor = new WP_HTML_Tag_Processor( 'This
` inside an HTML comment.
+ * - STYLE content is raw text.
+ * - TITLE content is plain text but character references are decoded.
+ * - TEXTAREA content is plain text but character references are decoded.
+ * - XMP (deprecated) content is raw text.
+ *
* ### Modifying HTML attributes for a found tag
*
* Once you've found the start of an opening tag you can modify
@@ -241,9 +288,13 @@
* double-quoted strings, meaning that attributes on input with single-quoted or
* unquoted values will appear in the output with double-quotes.
*
+ * scripts aren't processed
+ *
* @since 6.2.0
* @since 6.2.1 Fix: Support for various invalid comments; attribute updates are case-insensitive.
* @since 6.3.2 Fix: Skip HTML-like content inside rawtext elements such as STYLE.
+ * @since 6.5.0 Pauses processor when input ends in an incomplete syntax token.
+ * Introduces "special" elements which act like void elements, e.g. STYLE.
*/
class WP_HTML_Tag_Processor {
/**
@@ -316,6 +367,20 @@ class WP_HTML_Tag_Processor {
*/
private $stop_on_tag_closers;
+ /**
+ * What the parser is up to.
+ *
+ * @since {WP_VERSION}
+ *
+ * @see WP_HTML_Tag_Processor::STATE_READY
+ * @see WP_HTML_Tag_Processor::STATE_COMPLETE
+ * @see WP_HTML_Tag_Processor::STATE_INCOMPLETE
+ * @see WP_HTML_Tag_Processor::STATE_MATCHED_TAG
+ *
+ * @var string
+ */
+ private $parser_state = self::STATE_READY;
+
/**
* How many bytes from the original HTML document have been read and parsed.
*
@@ -544,6 +609,7 @@ public function __construct( $html ) {
* Finds the next tag matching the $query.
*
* @since 6.2.0
+ * @since 6.5.0 No longer processes incomplete tokens at end of document as text; pauses the processor instead.
*
* @param array|string|null $query {
* Optional. Which tag name to find, having which class, etc. Default is to find any tag.
@@ -562,86 +628,129 @@ public function next_tag( $query = null ) {
$already_found = 0;
do {
- if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ if ( false === $this->next_token() ) {
return false;
}
- // Find the next tag if it exists.
- if ( false === $this->parse_next_tag() ) {
- $this->bytes_already_parsed = strlen( $this->html );
-
- return false;
- }
-
- // Parse all of its attributes.
- while ( $this->parse_next_attribute() ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
continue;
}
- // Ensure that the tag closes before the end of the document.
- if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
- return false;
+ if ( $this->matches() ) {
+ ++$already_found;
}
+ } while ( $already_found < $this->sought_match_offset );
- $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed );
- if ( false === $tag_ends_at ) {
- return false;
- }
- $this->token_length = $tag_ends_at - $this->token_starts_at;
- $this->bytes_already_parsed = $tag_ends_at;
+ return true;
+ }
- // Finally, check if the parsed tag and its attributes match the search query.
- if ( $this->matches() ) {
- ++$already_found;
+ public function next_token() {
+ $this->get_updated_html();
+ $was_at = $this->bytes_already_parsed;
+
+ // Don't proceed if there's nothing more to scan.
+ if (
+ self::STATE_COMPLETE === $this->parser_state ||
+ self::STATE_INCOMPLETE === $this->parser_state
+ ) {
+ return false;
+ }
+
+ /*
+ * The next step in the parsing loop determines the parsing state;
+ * clear it so that state doesn't linger from the previous step.
+ */
+ $this->parser_state = self::STATE_READY;
+
+ if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_COMPLETE;
+ return false;
+ }
+
+ // Find the next tag if it exists.
+ if ( false === $this->parse_next_tag() ) {
+ if ( self::STATE_INCOMPLETE === $this->parser_state ) {
+ $this->bytes_already_parsed = $was_at;
}
- /*
- * For non-DATA sections which might contain text that looks like HTML tags but
- * isn't, scan with the appropriate alternative mode. Looking at the first letter
- * of the tag name as a pre-check avoids a string allocation when it's not needed.
- */
- $t = $this->html[ $this->tag_name_starts_at ];
- if (
- ! $this->is_closing_tag &&
+ return false;
+ }
+
+ // Parse all of its attributes.
+ while ( $this->parse_next_attribute() ) {
+ continue;
+ }
+
+ // Ensure that the tag closes before the end of the document.
+ if (
+ self::STATE_INCOMPLETE === $this->parser_state ||
+ $this->bytes_already_parsed >= strlen( $this->html )
+ ) {
+ // Does this appropriately clear state (parsed attributes)?
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ }
+
+ $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed );
+ if ( false === $tag_ends_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ }
+ $this->parser_state = self::STATE_MATCHED_TAG;
+ $this->token_length = $tag_ends_at - $this->token_starts_at;
+ $this->bytes_already_parsed = $tag_ends_at;
+
+ /*
+ * For non-DATA sections which might contain text that looks like HTML tags but
+ * isn't, scan with the appropriate alternative mode. Looking at the first letter
+ * of the tag name as a pre-check avoids a string allocation when it's not needed.
+ */
+ $t = $this->html[ $this->tag_name_starts_at ];
+ if (
+ ! $this->is_closing_tag &&
+ (
+ 'i' === $t || 'I' === $t ||
+ 'n' === $t || 'N' === $t ||
+ 's' === $t || 'S' === $t ||
+ 't' === $t || 'T' === $t ||
+ 'x' === $t || 'X' === $t
+ )
+ ) {
+ $tag_name = $this->get_tag();
+
+ if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ } elseif (
+ ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) &&
+ ! $this->skip_rcdata( $tag_name )
+ ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ } elseif (
(
- 'i' === $t || 'I' === $t ||
- 'n' === $t || 'N' === $t ||
- 's' === $t || 'S' === $t ||
- 't' === $t || 'T' === $t
- ) ) {
- $tag_name = $this->get_tag();
-
- if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) {
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- } elseif (
- ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) &&
- ! $this->skip_rcdata( $tag_name )
- ) {
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- } elseif (
- (
- 'IFRAME' === $tag_name ||
- 'NOEMBED' === $tag_name ||
- 'NOFRAMES' === $tag_name ||
- 'NOSCRIPT' === $tag_name ||
- 'STYLE' === $tag_name
- ) &&
- ! $this->skip_rawtext( $tag_name )
- ) {
- /*
- * "XMP" should be here too but its rules are more complicated and require the
- * complexity of the HTML Processor (it needs to close out any open P element,
- * meaning it can't be skipped here or else the HTML Processor will lose its
- * place). For now, it can be ignored as it's a rare HTML tag in practice and
- * any normative HTML should be using PRE instead.
- */
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- }
+ 'IFRAME' === $tag_name ||
+ 'NOEMBED' === $tag_name ||
+ 'NOFRAMES' === $tag_name ||
+ 'STYLE' === $tag_name ||
+ 'XMP' === $tag_name
+ ) &&
+ ! $this->skip_rawtext( $tag_name )
+ ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
}
- } while ( $already_found < $this->sought_match_offset );
+ }
return true;
}
@@ -664,6 +773,10 @@ public function next_tag( $query = null ) {
* @since 6.4.0
*/
public function class_list() {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
+ return;
+ }
+
/** @var string $class contains the string value of the class attribute, with character references decoded. */
$class = $this->get_attribute( 'class' );
@@ -719,7 +832,7 @@ public function class_list() {
* @return bool|null Whether the matched tag contains the given class name, or null if not matched.
*/
public function has_class( $wanted_class ) {
- if ( ! $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return null;
}
@@ -816,7 +929,8 @@ public function has_class( $wanted_class ) {
* @return bool Whether the bookmark was successfully created.
*/
public function set_bookmark( $name ) {
- if ( null === $this->tag_name_starts_at ) {
+ // It only makes sense to set a bookmark if the parser has paused on a concrete token.
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return false;
}
@@ -895,7 +1009,6 @@ private function skip_rcdata( $tag_name ) {
// Fail if there is no possible tag closer.
if ( false === $at || ( $at + $tag_length ) >= $doc_length ) {
- $this->bytes_already_parsed = $doc_length;
return false;
}
@@ -923,6 +1036,10 @@ private function skip_rcdata( $tag_name ) {
$at += $tag_length;
$this->bytes_already_parsed = $at;
+ if ( $at >= strlen( $html ) ) {
+ return false;
+ }
+
/*
* Ensure that the tag name terminates to avoid matching on
* substrings of a longer tag name. For example, the sequence
@@ -1107,6 +1224,10 @@ private function parse_next_tag() {
while ( false !== $at && $at < $doc_length ) {
$at = strpos( $html, '<', $at );
+ /*
+ * This does not imply an incomplete parse; it indicates that there
+ * can be nothing left in the document other than a #text node.
+ */
if ( false === $at ) {
return false;
}
@@ -1148,6 +1269,8 @@ private function parse_next_tag() {
* the document. There is nothing left to parse.
*/
if ( $at + 1 >= strlen( $html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1168,6 +1291,8 @@ private function parse_next_tag() {
$closer_at = $at + 4;
// If it's not possible to close the comment then there is nothing more to scan.
if ( strlen( $html ) <= $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1188,6 +1313,8 @@ private function parse_next_tag() {
while ( ++$closer_at < strlen( $html ) ) {
$closer_at = strpos( $html, '--', $closer_at );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1220,6 +1347,8 @@ private function parse_next_tag() {
) {
$closer_at = strpos( $html, ']]>', $at + 9 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1244,6 +1373,8 @@ private function parse_next_tag() {
) {
$closer_at = strpos( $html, '>', $at + 9 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1276,6 +1407,8 @@ private function parse_next_tag() {
if ( '?' === $html[ $at + 1 ] ) {
$closer_at = strpos( $html, '>', $at + 2 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1292,6 +1425,8 @@ private function parse_next_tag() {
if ( $this->is_closing_tag ) {
$closer_at = strpos( $html, '>', $at + 3 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1316,6 +1451,8 @@ private function parse_next_attribute() {
// Skip whitespace and slashes.
$this->bytes_already_parsed += strspn( $this->html, " \t\f\r\n/", $this->bytes_already_parsed );
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1338,11 +1475,15 @@ private function parse_next_attribute() {
$attribute_name = substr( $this->html, $attribute_start, $name_length );
$this->bytes_already_parsed += $name_length;
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
$this->skip_whitespace();
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1351,6 +1492,8 @@ private function parse_next_attribute() {
++$this->bytes_already_parsed;
$this->skip_whitespace();
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1377,6 +1520,8 @@ private function parse_next_attribute() {
}
if ( $attribute_end >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1443,7 +1588,6 @@ private function skip_whitespace() {
* @since 6.2.0
*/
private function after_tag() {
- $this->get_updated_html();
$this->token_starts_at = null;
$this->token_length = null;
$this->tag_name_starts_at = null;
@@ -1786,6 +1930,10 @@ private static function sort_start_ascending( $a, $b ) {
* @return string|boolean|null Value of enqueued update if present, otherwise false.
*/
private function get_enqueued_attribute_value( $comparable_name ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
+ return false;
+ }
+
if ( ! isset( $this->lexical_updates[ $comparable_name ] ) ) {
return false;
}
@@ -1853,7 +2001,7 @@ private function get_enqueued_attribute_value( $comparable_name ) {
* @return string|true|null Value of attribute or `null` if not available. Boolean attributes return `true`.
*/
public function get_attribute( $name ) {
- if ( null === $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return null;
}
@@ -1933,7 +2081,10 @@ public function get_attribute( $name ) {
* @return array|null List of attribute names, or `null` when no tag opener is matched.
*/
public function get_attribute_names_with_prefix( $prefix ) {
- if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return null;
}
@@ -1965,7 +2116,7 @@ public function get_attribute_names_with_prefix( $prefix ) {
* @return string|null Name of currently matched tag in input HTML, or `null` if none found.
*/
public function get_tag() {
- if ( null === $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return null;
}
@@ -1992,7 +2143,7 @@ public function get_tag() {
* @return bool Whether the currently matched tag contains the self-closing flag.
*/
public function has_self_closing_flag() {
- if ( ! $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return false;
}
@@ -2024,7 +2175,10 @@ public function has_self_closing_flag() {
* @return bool Whether the current tag is a tag closer.
*/
public function is_tag_closer() {
- return $this->is_closing_tag;
+ return (
+ self::STATE_MATCHED_TAG === $this->parser_state &&
+ $this->is_closing_tag
+ );
}
/**
@@ -2044,7 +2198,10 @@ public function is_tag_closer() {
* @return bool Whether an attribute value was set.
*/
public function set_attribute( $name, $value ) {
- if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2177,7 +2334,10 @@ public function set_attribute( $name, $value ) {
* @return bool Whether an attribute was removed.
*/
public function remove_attribute( $name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2254,13 +2414,14 @@ public function remove_attribute( $name ) {
* @return bool Whether the class was set to be added.
*/
public function add_class( $class_name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
- if ( null !== $this->tag_name_starts_at ) {
- $this->classname_updates[ $class_name ] = self::ADD_CLASS;
- }
+ $this->classname_updates[ $class_name ] = self::ADD_CLASS;
return true;
}
@@ -2274,7 +2435,10 @@ public function add_class( $class_name ) {
* @return bool Whether the class was set to be removed.
*/
public function remove_class( $class_name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2480,4 +2644,9 @@ private function matches() {
return true;
}
+
+ const STATE_READY = 'READY: The parser is waiting for a state transition; it may not have started, or it may have been interrupted, or it may be waiting to restart after pausing.';
+ const STATE_COMPLETE = 'COMPLETE: The parser has reached the end of the document without truncating any possible tokens. There is nothing left to scan.';
+ const STATE_INCOMPLETE = 'INCOMPLETE: The parser has reached the end of the document but it appears as thought the HTML is truncated inside a token. It has backed up to the last-known complete state and will not continue parsing.';
+ const STATE_MATCHED_TAG = 'MATCHED_TAG: The parser has found a tag and paused to allow reading from and modifying its attributes.';
}
diff --git a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php
index 4469f90c4f276..f500512deeacd 100644
--- a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php
+++ b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php
@@ -1758,9 +1758,8 @@ public function test_setting_a_boolean_attribute_to_a_string_value_adds_explicit
* @covers WP_HTML_Tag_Processor::next_tag
*/
public function test_unclosed_script_tag_should_not_cause_an_infinite_loop() {
- $p = new WP_HTML_Tag_Processor( ' array( ' array( '