From a491befb052b310d22746949bb4d7153b01887f9 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 23 Jul 2026 12:27:44 +0200 Subject: [PATCH 01/77] CSS API: Add CSS token processor foundations --- .../css-api/class-wp-css-builder.php | 261 + .../css-api/class-wp-css-token-processor.php | 1863 ++++++ src/wp-settings.php | 2 + .../phpunit/data/css-api/css-test-cases.json | 4988 +++++++++++++++++ tests/phpunit/tests/css-api/wpCssBuilder.php | 150 + .../tests/css-api/wpCssTokenProcessor.php | 2809 ++++++++++ 6 files changed, 10073 insertions(+) create mode 100644 src/wp-includes/css-api/class-wp-css-builder.php create mode 100644 src/wp-includes/css-api/class-wp-css-token-processor.php create mode 100644 tests/phpunit/data/css-api/css-test-cases.json create mode 100644 tests/phpunit/tests/css-api/wpCssBuilder.php create mode 100644 tests/phpunit/tests/css-api/wpCssTokenProcessor.php diff --git a/src/wp-includes/css-api/class-wp-css-builder.php b/src/wp-includes/css-api/class-wp-css-builder.php new file mode 100644 index 0000000000000..35250335da661 --- /dev/null +++ b/src/wp-includes/css-api/class-wp-css-builder.php @@ -0,0 +1,261 @@ += 0x80 ) { + $result .= $value[ $i ]; + continue; + } + + // ASCII letters and underscore: always valid in idents. + if ( + ( $byte >= 0x41 && $byte <= 0x5A ) || // A-Z + ( $byte >= 0x61 && $byte <= 0x7A ) || // a-z + 0x5F === $byte // _ + ) { + $result .= $value[ $i ]; + continue; + } + + // Hyphen: valid in idents, but check for hyphen-digit at start. + if ( 0x2D === $byte ) { + // Hyphen at position 0 followed by a digit at position 1: escape the digit. + if ( 0 === $i && $i + 1 < $length && ord( $value[ $i + 1 ] ) >= 0x30 && ord( $value[ $i + 1 ] ) <= 0x39 ) { + $result .= '-'; + ++$i; + $result .= sprintf( '\\%X ', ord( $value[ $i ] ) ); + continue; + } + $result .= '-'; + continue; + } + + // Digits: valid except at position 0. + if ( $byte >= 0x30 && $byte <= 0x39 ) { + if ( 0 === $i ) { + $result .= sprintf( '\\%X ', $byte ); + } else { + $result .= $value[ $i ]; + } + continue; + } + + // Everything else: hex-escape. + $result .= sprintf( '\\%X ', $byte ); + } + + return $result; + } + + /** + * Create a quoted CSS string from a plain PHP string value. + * + * Example: + * $value = 'CSS & a ""; + * + * CSS strings are quoted many characters that are problematic in HTML + * or may be complicated for rudimentary CSS or HTML processors to handle + * are encoded using Unicode escape sequences. + * + * @see https://www.w3.org/TR/css-syntax-3/#escaping + */ + public static function string( string $value ): string { + $value = wp_scrub_utf8( $value ); + $escaped = strtr( + $value, + array( + // Escape existing backslashes to prevent unintentional escapes in result. + '\\' => '\\5C ', + + // Pre-processing replaces NULLs and some newlines. Replace and escape as necessary. + "\0" => "\u{FFFD}", + + // Normalize and replace newlines. https://www.w3.org/TR/css-syntax-3/#input-preprocessing + "\r\n" => '\\A ', + "\r" => '\\A ', + "\f" => '\\A ', + + // Newlines must be escaped in CSS strings. + "\n" => '\\A ', + + // Arbitrary characters for Unicode escaping: + + // HTML syntax may be problematic. + '<' => '\\3C ', + '>' => '\\3E ', + '&' => '\\26 ', + + // CSS syntax may be problematic. + ',' => '\\2C ', + ';' => '\\3B ', + '{' => '\\7B ', + '}' => '\\7D ', + '"' => '\\22 ', + "'" => '\\27 ', + ) + ); + return "\"{$escaped}\""; + } + + public static function normalize_and_escape_css( string $css ): string { + $css = wp_scrub_utf8( $css ); + $processor = WP_CSS_Token_Processor::create( $css ); + if ( null === $processor ) { + return ''; + } + + $normalized_css = ''; + + while ( $processor->next_token() ) { + switch ( $processor->get_token_type() ) { + + // Basic punctuation: + case WP_CSS_Token_Processor::TOKEN_SEMICOLON: + $normalized_css .= ';'; + break; + case WP_CSS_Token_Processor::TOKEN_COMMA: + $normalized_css .= ','; + break; + case WP_CSS_Token_Processor::TOKEN_WHITESPACE: + $normalized_css .= ' '; + break; + case WP_CSS_Token_Processor::TOKEN_COLON: + $normalized_css .= ':'; + break; + + // Paired punctuation: + case WP_CSS_Token_Processor::TOKEN_LEFT_BRACE: + $normalized_css .= '{'; + break; + case WP_CSS_Token_Processor::TOKEN_RIGHT_BRACE: + $normalized_css .= '}'; + break; + case WP_CSS_Token_Processor::TOKEN_LEFT_PAREN: + $normalized_css .= '('; + break; + case WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN: + $normalized_css .= ')'; + break; + case WP_CSS_Token_Processor::TOKEN_LEFT_BRACKET: + $normalized_css .= '['; + break; + case WP_CSS_Token_Processor::TOKEN_RIGHT_BRACKET: + $normalized_css .= ']'; + break; + + // "@" + ident + case WP_CSS_Token_Processor::TOKEN_AT_KEYWORD: + $normalized_css .= '@' . self::ident( $processor->get_token_value() ); + break; + + // ident + "(" + case WP_CSS_Token_Processor::TOKEN_FUNCTION: + $normalized_css .= self::ident( $processor->get_token_value() ) . '('; + break; + + /* + * Hash tokens are not idents but their value can be escaped as such. + * + * ‖→ "#" →─┐ ┌──────────────────────────────┐ ┌─→‖ + * ├─→─┤ a-z A-Z 0-9 _ - or non-ASCII ├─→─┤ + * │ └──────────────────────────────┘ │ + * │ ┌──────────────────────────────┐ │ + * ├─→─┤ escape ├─→─┤ + * │ └──────────────────────────────┘ │ + * └──────────────────←───────────────────┘ + */ + case WP_CSS_Token_Processor::TOKEN_HASH: + $normalized_css .= '#' . self::ident( $processor->get_token_value() ); + break; + + case WP_CSS_Token_Processor::TOKEN_DIMENSION: + $normalized_css .= $processor->get_token_value() . $processor->get_token_unit(); + break; + + case WP_CSS_Token_Processor::TOKEN_PERCENTAGE: + $normalized_css .= "%{$processor->get_token_value()}"; + break; + + case WP_CSS_Token_Processor::TOKEN_NUMBER: + $normalized_css .= $processor->get_token_value(); + break; + + case WP_CSS_Token_Processor::TOKEN_DELIM: + $normalized_css .= $processor->get_token_value(); + break; + + case WP_CSS_Token_Processor::TOKEN_IDENT: + $normalized_css .= self::ident( $processor->get_token_value() ); + break; + + case WP_CSS_Token_Processor::TOKEN_STRING: + var_dump( $processor->get_token_value() ); + $normalized_css .= self::string( $processor->get_token_value() ); + break; + + // Keep or strip comments? + case WP_CSS_Token_Processor::TOKEN_COMMENT: + $normalized_css .= substr( $css, $processor->get_token_start(), $processor->get_token_length() ); + break; + + /** + * A is an open string that reaches a newline. + * + * @see https://www.w3.org/TR/css-syntax-3/#consume-string-token + * + * @see https://www.w3.org/TR/css-syntax-3/#preserved-tokens + * > Note: The tokens <}-token>s, <)-token>s, <]-token>, , and are always parse errors, but they are preserved in the token stream by this specification to allow other specs, such as Media Queries, to define more fine-grained error-handling than just dropping an entire declaration or block. + */ + case WP_CSS_Token_Processor::TOKEN_BAD_STRING: + $normalized_css .= substr( $css, $processor->get_token_start(), $processor->get_token_length() ) . "\n"; + break; + + case WP_CSS_Token_Processor::TOKEN_URL: + case WP_CSS_Token_Processor::TOKEN_BAD_URL: + case WP_CSS_Token_Processor::TOKEN_CDC: + case WP_CSS_Token_Processor::TOKEN_CDO: + default: + throw new Error( 'unhandled token type ' . $processor->get_token_type() . ' with value ' . var_export( $processor->get_token_value(), true ) ); + } + } + + return strtr( + $normalized_css, + array( + ' ' => '␠', + "\t" => "␉\t", + "\n" => "␊\n", + ) + ); + } +} diff --git a/src/wp-includes/css-api/class-wp-css-token-processor.php b/src/wp-includes/css-api/class-wp-css-token-processor.php new file mode 100644 index 0000000000000..74c548dc1ec3a --- /dev/null +++ b/src/wp-includes/css-api/class-wp-css-token-processor.php @@ -0,0 +1,1863 @@ + Replace any U+000D CARRIAGE RETURN (CR) code points, U+000C FORM FEED (FF) + * > code points, or pairs of U+000D CARRIAGE RETURN (CR) followed by U+000A LINE + * > FEED (LF) in input by a single U+000A LINE FEED (LF) code point. + * > Replace any U+0000 NULL or surrogate code points in input with U+FFFD REPLACEMENT + * > CHARACTER (�). + * + * This processor delays normalization as much as possible. That keeps the raw byte + * positions intact for accurate rewrites while still letting consumers ask for a + * normalized token when they need one. + * + * ### No EOF token + * + * The EOF token is a CSS parsing concept, not CSS tokenization concept. Therefore, + * this processor does not produce it. + * + * ### UTF-8 handling + * + * Only UTF-8 strings are supported. Invalid sequences are replaced with U+FFFD (�) + * using the maximal subpart approach described in + * https://www.unicode.org/versions/Unicode9.0.0/ch03.pdf, section 3.9 Best Practices + * for Using U+FFFD. + * + * ## Usage + * + * Basic iteration: + * + * $css = 'width: 10px;'; + * $processor = WP_CSS_Token_Processor::create( $css ); + * while ( $processor->next_token() ) { + * echo $processor->get_normalized_token(); + * } + * // Outputs: + * // width: 10px; + * + * Rewriting a URL while keeping the rest of the stylesheet intact: + * + * $css = 'background: url(old.jpg) center / cover;'; + * $processor = WP_CSS_Token_Processor::create( $css ); + * while ( $processor->next_token() ) { + * if ( WP_CSS_Token_Processor::TOKEN_URL === $processor->get_token_type() ) { + * $processor->set_value( 'uploads/new.jpg' ); + * } + * } + * $result = $processor->get_updated_css(); + * // background: url(uploads/new.jpg) center / cover; + * + * Gathering diagnostics with byte offsets: + * + * $css = "color: red;\ncolor: re\nd;"; + * $processor = WP_CSS_Token_Processor::create( $css ); + * $bad_strings = array(); + * while ( $processor->next_token() ) { + * if ( WP_CSS_Token_Processor::TOKEN_BAD_STRING === $processor->get_token_type() ) { + * $bad_strings[] = array( + * 'start' => $processor->get_token_start(), + * 'length' => $processor->get_token_length(), + * 'value' => $processor->get_unnormalized_token(), + * ); + * } + * } + * + * @see https://www.w3.org/TR/css-syntax-3/#tokenization + */ +class WP_CSS_Token_Processor { + /** + * Token type constants matching the CSS Syntax Level 3 specification. + * + * @see https://www.w3.org/TR/css-syntax-3/#tokenization + */ + public const TOKEN_WHITESPACE = 'whitespace-token'; + public const TOKEN_COMMENT = 'comment'; + public const TOKEN_STRING = 'string-token'; + + /** + * BAD-STRING tokens occur when a string contains an unescaped newline. + * + * Valid strings: "hello", 'world', "line1\Aline2" (escaped newline) + * Invalid (produces bad-string): "hello + * world" (literal newline breaks the string) + * + * The processor stops at the newline and produces a bad-string token for error recovery. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-bad-string-token + */ + public const TOKEN_BAD_STRING = 'bad-string-token'; + public const TOKEN_HASH = 'hash-token'; + public const HASH_TOKEN_ID = 'id'; + public const HASH_TOKEN_UNRESTRICTED = 'unrestricted'; + public const TOKEN_DELIM = 'delim-token'; + public const TOKEN_NUMBER = 'number-token'; + public const TOKEN_PERCENTAGE = 'percentage-token'; + public const TOKEN_DIMENSION = 'dimension-token'; + public const TOKEN_AT_KEYWORD = 'at-keyword-token'; + public const TOKEN_COLON = 'colon-token'; + public const TOKEN_SEMICOLON = 'semicolon-token'; + public const TOKEN_COMMA = 'comma-token'; + public const TOKEN_LEFT_PAREN = '(-token'; + public const TOKEN_RIGHT_PAREN = ')-token'; + public const TOKEN_LEFT_BRACKET = '[-token'; + public const TOKEN_RIGHT_BRACKET = ']-token'; + public const TOKEN_LEFT_BRACE = '{-token'; + public const TOKEN_RIGHT_BRACE = '}-token'; + public const TOKEN_FUNCTION = 'function-token'; + + /** + * URL tokens represent unquoted URLs in url() notation. + * + * For example, `url(image.jpg)` is a URL token. + * + * Quoted URLs like `url( "https://example.com" )` are handled as a function + * token, _not_ a URL token. + * + * Bad URL tokens are created when invalid characters are encountered in + * a URL token. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-url-token + */ + public const TOKEN_URL = 'url-token'; + + /** + * BAD-URL tokens occur when a URL contains invalid characters. + * + * Invalid characters: quotes ("), apostrophes ('), parentheses (() + * Example invalid: url(image(.jpg) or url(image".jpg) + * + * When detected, the processor consumes everything up to ) or EOF. + * This prevents the bad URL from breaking subsequent tokens. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-bad-url-token + */ + public const TOKEN_BAD_URL = 'bad-url-token'; + + /** + * Identifier tokens, such as `color`, `margin-top`, `red`, + * `inherit`, `--my-var`, `\x-escaped`, `über` (Unicode), etc. + * + * There are restrictions on the codepoints that start or are contained in + * an identifier, and identifiers may contain escape sequences. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-ident-token + */ + public const TOKEN_IDENT = 'ident-token'; + + /** + * CDC (Comment Delimiter Close) token: --> + * + * Legacy token from when CSS was embedded in HTML + * + * Modern CSS no longer needs these, but they're preserved for compatibility. + * In stylesheets, they're typically treated like whitespace. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-CDC-token + */ + public const TOKEN_CDC = 'CDC-token'; + + /** + * CDO (Comment Delimiter Open) token: ) + * + * Comment Delimiter Close - legacy HTML comment syntax in CSS. + * + * @see https://www.w3.org/TR/css-syntax-3/#CDC-token-diagram + */ + if ( + $this->at + 2 < $this->length && + '-' === $this->css[ $this->at + 1 ] && + '>' === $this->css[ $this->at + 2 ] + ) { + // Consume them and return a . + $this->at += 3; + $this->token_type = self::TOKEN_CDC; + $this->token_length = 3; + return true; + } + + // Otherwise, if the input stream starts with an ident sequence, + // reconsume the current input code point, consume an ident-like + // token, and return it. + if ( $this->check_if_3_code_points_start_an_ident_sequence( $this->at ) ) { + return $this->consume_ident_like(); + } + + // Otherwise, return a with its value set to the current input code point. + ++$this->at; + $this->token_type = self::TOKEN_DELIM; + $this->token_length = 1; + return true; + } + + /* + * U+003C LESS-THAN SIGN (<) + * If followed by !--, this is a CDO token (\n", + "tokens": [ + { + "type": "CDC-token", + "raw": "-->", + "startIndex": 0, + "endIndex": 3, + "normalized": "-->", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0001": { + "css": "foo\n", + "tokens": [ + { + "type": "ident-token", + "raw": "foo", + "startIndex": 0, + "endIndex": 3, + "normalized": "foo", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0002": { + "css": "--\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--", + "startIndex": 0, + "endIndex": 2, + "normalized": "--", + "value": "--" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0003": { + "css": "--0\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--0", + "startIndex": 0, + "endIndex": 3, + "normalized": "--0", + "value": "--0" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0004": { + "css": "-\\\n", + "tokens": [ + { + "type": "delim-token", + "raw": "-", + "startIndex": 0, + "endIndex": 1, + "normalized": "-", + "value": "-" + }, + { + "type": "delim-token", + "raw": "\\", + "startIndex": 1, + "endIndex": 2, + "normalized": "\\", + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0005": { + "css": "-\\ \n", + "tokens": [ + { + "type": "ident-token", + "raw": "-\\ ", + "startIndex": 0, + "endIndex": 3, + "normalized": "- ", + "value": "- " + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0006": { + "css": "--💅\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--💅", + "startIndex": 0, + "endIndex": 6, + "normalized": "--💅", + "value": "--💅" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0007": { + "css": "-§\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-§", + "startIndex": 0, + "endIndex": 3, + "normalized": "-§", + "value": "-§" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0008": { + "css": "-×\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-×", + "startIndex": 0, + "endIndex": 3, + "normalized": "-×", + "value": "-×" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0009": { + "css": "--a𐀀\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--a𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "--a𐀀", + "value": "--a𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0001": { + "css": "url(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url(foo)", + "startIndex": 0, + "endIndex": 8, + "normalized": "url(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0002": { + "css": "\\75 Rl(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "\\75 Rl(foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "uRl(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0003": { + "css": "uR\\6c (foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "uR\\6c (foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "uRl(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0004": { + "css": "url('foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 4, + "endIndex": 9, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 9, + "endIndex": 10, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 10, + "endIndex": 11, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0005": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 5, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 5, + "endIndex": 10, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 10, + "endIndex": 11, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0006": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 6, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 6, + "endIndex": 11, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 11, + "endIndex": 12, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 12, + "endIndex": 13, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0007": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 7, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 7, + "endIndex": 12, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 12, + "endIndex": 13, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0008": { + "css": "not-url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "not-url(", + "startIndex": 0, + "endIndex": 8, + "normalized": "not-url(", + "value": "not-url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 8, + "endIndex": 11, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 11, + "endIndex": 16, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 16, + "endIndex": 17, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 17, + "endIndex": 18, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0009": { + "css": "url( foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url( foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "url( foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-curly-bracket/0001": { + "css": "{\n", + "tokens": [ + { + "type": "{-token", + "raw": "{", + "startIndex": 0, + "endIndex": 1, + "normalized": "{", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-parenthesis/0001": { + "css": "(\n", + "tokens": [ + { + "type": "(-token", + "raw": "(", + "startIndex": 0, + "endIndex": 1, + "normalized": "(", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-square-bracket/0001": { + "css": "[\n", + "tokens": [ + { + "type": "[-token", + "raw": "[", + "startIndex": 0, + "endIndex": 1, + "normalized": "[", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/less-than/0001": { + "css": "<\n", + "tokens": [ + { + "type": "delim-token", + "raw": "<", + "startIndex": 0, + "endIndex": 1, + "normalized": "<", + "value": "<" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/less-than/0002": { + "css": "', + WP_CSS_Token_Processor::TOKEN_BAD_STRING => "\"\n", + WP_CSS_Token_Processor::TOKEN_BAD_URL => 'url(()', + ); + + /** + * Closing token type of each token type that opens a block. + * + * @since 7.2.0 + * + * @var array + */ + private const BLOCK_CLOSERS = array( + WP_CSS_Token_Processor::TOKEN_FUNCTION => WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, + WP_CSS_Token_Processor::TOKEN_LEFT_PAREN => WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, + WP_CSS_Token_Processor::TOKEN_LEFT_BRACKET => WP_CSS_Token_Processor::TOKEN_RIGHT_BRACKET, + WP_CSS_Token_Processor::TOKEN_LEFT_BRACE => WP_CSS_Token_Processor::TOKEN_RIGHT_BRACE, + ); + + /** + * Serializes tokens as CSS text that reads as the same tokens. + * + * This follows CSS Syntax Level 3 §9 Serialization: + * + * > The only requirement for serialization is that it must "round-trip" with parsing, + * > that is, parsing the stylesheet must produce the same data structures as + * > parsing, serializing, and parsing again, + * > except for consecutive s, + * > which may be collapsed into a single token. + * + * Comment tokens are dropped. Each run of whitespace, including a run + * broken by comments, is written as one space; a run is never removed. + * + * Tokens are written from their decoded values. Identifiers, function + * names, at-keywords, hash names, dimension units, strings and url values + * are escaped again, so `\66 oo` becomes `foo` and `"a&b"` becomes `"a\26 b"`. + * + * > A containing U+005C REVERSE SOLIDUS (\) + * > must be serialized as U+005C REVERSE SOLIDUS + * > followed by a newline. + * > (The tokenizer only ever emits such a token followed by a + * > that starts with a newline.) + * + * > A with the "unrestricted" type flag may not need + * > as much escaping as the same token with the "id" type flag. + * + * A hash with the "id" flag is escaped as an identifier, so `#\31 a` stays + * `#\31 a`. A hash with the "unrestricted" flag may start with a digit or + * `-`, so `#1a` stays `#1a`. + * + * > The unit of a may need escaping + * > to disambiguate with scientific notation. + * + * A unit that starts with `e`, an optional `-`, and a digit has the `e` + * escaped, so the unit `e3` is written `1\65 3`, not `1e3`. + * + * > For any consecutive pair of tokens, + * > if the first token shows up in the row headings of the following table, + * > and the second token shows up in the column headings, + * > and there’s a ✗ in the cell denoted by the intersection of the chosen row and column, + * > the pair of tokens must be serialized with a comment between them. + * > + * > Single characters in the row and column headings represent a with that value, + * > except for "(", + * > which represents a (-token. + * + * ┌────────────┬───────┬──────────┬─────┬─────────┬───┬────────┬────────────┬───────────┬─────┬───┬───┬───┐ + * │ │ ident │ function │ url │ bad url │ - │ number │ percentage │ dimension │ CDC │ ( │ * │ % │ + * ├────────────┼───────┼──────────┼─────┼─────────┼───┼────────┼────────────┼───────────┼─────┼───┼───┼───┤ + * │ ident │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ + * │ at-keyword │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ hash │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ dimension │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ # │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ - │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ number │ ✗ │ ✗ │ ✗ │ ✗ │ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ ✗ │ + * │ @ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ ✗ │ │ │ │ + * │ . │ │ │ │ │ │ ✗ │ ✗ │ ✗ │ │ │ │ │ + * │ + │ │ │ │ │ │ ✗ │ ✗ │ ✗ │ │ │ │ │ + * │ / │ │ │ │ │ │ │ │ │ │ │ ✗ │ │ + * └────────────┴───────┴──────────┴─────┴─────────┴───┴────────┴────────────┴───────────┴─────┴───┴───┴───┘ + * + * Comments are not preserved, so an empty CSS comment is written between + * the tokens of a marked pair when no whitespace is between them. + * + * Bad-string and bad-url tokens have no value. A bad string is written as + * a quote followed by a newline, where the tokenizer ends one; a bad url is + * written as `url(()`. + * + * Strings and urls cut off by the end of the input are written closed. + * Blocks still open at the end are closed in reverse order, so `a(b[c` is + * written `a(b[c])`. + * + * Example: + * + * $processor = WP_CSS_Token_Processor::create( 'url( a&b ) calc(1px + 2px' ); + * $tokens = array(); + * while ( $processor->next_token() ) { + * $tokens[] = $processor->get_token(); + * } + * WP_CSS_Token_Serializer::serialize( $tokens ); // 'url(a\26 b) calc(1px + 2px)' + * + * @link https://www.w3.org/TR/css-syntax-3/#serialization + * + * @since 7.2.0 + * + * @param iterable $tokens Tokens read by WP_CSS_Token_Processor::get_token(). + * @return string CSS text. + */ + public static function serialize( iterable $tokens ): string { + $output = ''; + $closers = array(); + + // Pair key of the last token written, or the whitespace type after a space. + $last = null; + + foreach ( $tokens as $token ) { + $type = $token->get_type(); + + if ( WP_CSS_Token_Processor::TOKEN_COMMENT === $type ) { + continue; + } + + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE === $type ) { + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE !== $last ) { + $output .= ' '; + $last = WP_CSS_Token_Processor::TOKEN_WHITESPACE; + } + continue; + } + + $key = WP_CSS_Token_Processor::TOKEN_DELIM === $type ? $token->get_value() : $type; + if ( null !== $last && isset( self::MARKED_PAIRS[ $last ] ) && in_array( $key, self::MARKED_PAIRS[ $last ], true ) ) { + $output .= '/**/'; + } + $last = $key; + + $output .= self::serialize_token( $token ); + + if ( isset( self::BLOCK_CLOSERS[ $type ] ) ) { + $closers[] = self::BLOCK_CLOSERS[ $type ]; + } elseif ( end( $closers ) === $type ) { + array_pop( $closers ); + } + } + + while ( array() !== $closers ) { + $output .= self::FIXED_TEXT[ array_pop( $closers ) ]; + } + + return $output; + } + + /** + * Serializes one token other than whitespace or a comment. + * + * @since 7.2.0 + * + * @param WP_CSS_Token $token Token. + * @return string CSS text. + */ + private static function serialize_token( WP_CSS_Token $token ): string { + $value = $token->get_value(); + + switch ( $token->get_type() ) { + case WP_CSS_Token_Processor::TOKEN_IDENT: + return self::serialize_ident( $value ); + + case WP_CSS_Token_Processor::TOKEN_FUNCTION: + return self::serialize_ident( $value ) . '('; + + case WP_CSS_Token_Processor::TOKEN_AT_KEYWORD: + return '@' . self::serialize_ident( $value ); + + case WP_CSS_Token_Processor::TOKEN_HASH: + return WP_CSS_Token_Processor::HASH_TOKEN_ID === $token->get_flag() + ? '#' . self::serialize_ident( $value ) + : '#' . self::escape_name_code_points( $value ); + + case WP_CSS_Token_Processor::TOKEN_STRING: + return self::quote_string( $value ); + + case WP_CSS_Token_Processor::TOKEN_URL: + return 'url(' . self::serialize_url_value( $value ) . ')'; + + case WP_CSS_Token_Processor::TOKEN_NUMBER: + return $value; + + case WP_CSS_Token_Processor::TOKEN_PERCENTAGE: + return $value . '%'; + + case WP_CSS_Token_Processor::TOKEN_DIMENSION: + $unit = self::serialize_ident( $token->get_unit() ); + // A unit that starts with `e`, an optional `-`, and a digit would read as an exponent. + if ( 1 === preg_match( '/^[eE]-?[0-9]/', $unit ) ) { + $unit = sprintf( '\\%X ', ord( $unit[0] ) ) . substr( $unit, 1 ); + } + return $value . $unit; + + case WP_CSS_Token_Processor::TOKEN_DELIM: + return '\\' === $value ? "\\\n" : $value; + + default: + return self::FIXED_TEXT[ $token->get_type() ]; + } + } + /** * Serializes a plain PHP string as CSS identifier text. * @@ -118,40 +439,148 @@ public static function serialize_ident( string $value ): string { * @return string Serialized quoted CSS string text. */ public static function serialize_string( string $value ): string { + // Normalize newlines. https://www.w3.org/TR/css-syntax-3/#input-preprocessing + $value = strtr( + $value, + array( + "\r\n" => "\n", + "\r" => "\n", + "\f" => "\n", + ) + ); + + return self::quote_string( $value ); + } + + /** + * Creates a quoted CSS string from a decoded value without normalizing newlines. + * + * A CR or FF in a token value comes from an escape such as `\D `, so it is + * escaped, not written as a newline. + * + * @since 7.2.0 + * + * @param string $value Decoded string value to encode as a quoted CSS string. + * @return string Serialized quoted CSS string text. + */ + private static function quote_string( string $value ): string { $value = wp_scrub_utf8( $value ); $escaped = strtr( $value, array( // Escape existing backslashes to prevent unintentional escapes in result. - '\\' => '\\5C ', + '\\' => '\\5C ', - // Pre-processing replaces NULLs and some newlines. Replace and escape as necessary. - "\0" => "\u{FFFD}", - - // Normalize and replace newlines. https://www.w3.org/TR/css-syntax-3/#input-preprocessing - "\r\n" => '\\A ', - "\r" => '\\A ', - "\f" => '\\A ', + // Pre-processing replaces NULLs. Replace as necessary. + "\0" => "\u{FFFD}", // Newlines must be escaped in CSS strings. - "\n" => '\\A ', + "\n" => '\\A ', + "\r" => '\\D ', + "\f" => '\\C ', // Arbitrary characters for Unicode escaping: // HTML syntax may be problematic. - '<' => '\\3C ', - '>' => '\\3E ', - '&' => '\\26 ', + '<' => '\\3C ', + '>' => '\\3E ', + '&' => '\\26 ', // CSS syntax may be problematic. - ',' => '\\2C ', - ';' => '\\3B ', - '{' => '\\7B ', - '}' => '\\7D ', - '"' => '\\22 ', - "'" => '\\27 ', + ',' => '\\2C ', + ';' => '\\3B ', + '{' => '\\7B ', + '}' => '\\7D ', + '"' => '\\22 ', + "'" => '\\27 ', ) ); return "\"{$escaped}\""; } + + /** + * Escapes every code point that cannot appear in an identifier. + * + * Letters, digits, `-`, `_` and code points U+0080 and above are written + * as they are; every other code point is hex-escaped. Unlike + * serialize_ident(), a leading digit or `-` is not escaped, as the name of + * a hash token with the "unrestricted" flag requires. + * + * @since 7.2.0 + * + * @param string $value Decoded name. + * @return string Escaped name. + */ + private static function escape_name_code_points( string $value ): string { + $value = wp_scrub_utf8( $value ); + $result = ''; + $length = strlen( $value ); + + for ( $i = 0; $i < $length; $i++ ) { + $byte = ord( $value[ $i ] ); + + if ( 0x00 === $byte ) { + $result .= "\u{FFFD}"; + continue; + } + + if ( + $byte >= 0x80 || + ( $byte >= 0x30 && $byte <= 0x39 ) || // 0-9 + ( $byte >= 0x41 && $byte <= 0x5A ) || // A-Z + ( $byte >= 0x61 && $byte <= 0x7A ) || // a-z + 0x5F === $byte || // _ + 0x2D === $byte // - + ) { + $result .= $value[ $i ]; + continue; + } + + $result .= sprintf( '\\%X ', $byte ); + } + + return $result; + } + + /** + * Escapes the value of a url token. + * + * The code points an unquoted url cannot contain are hex-escaped: quotes, + * parentheses, `\`, whitespace and non-printable code points. So are the + * code points serialize_string() escapes for HTML and CSS: `< > & , ; { }`. + * + * @see https://www.w3.org/TR/css-syntax-3/#consume-url-token + * + * @since 7.2.0 + * + * @param string $value Decoded url value. + * @return string Escaped url value, without `url(` and `)`. + */ + private static function serialize_url_value( string $value ): string { + $value = wp_scrub_utf8( $value ); + $result = ''; + $length = strlen( $value ); + + for ( $i = 0; $i < $length; $i++ ) { + $byte = ord( $value[ $i ] ); + + if ( 0x00 === $byte ) { + $result .= "\u{FFFD}"; + continue; + } + + if ( + $byte <= 0x20 || + 0x7F === $byte || + false !== strpos( '"\'()\\<>&,;{}', $value[ $i ] ) + ) { + $result .= sprintf( '\\%X ', $byte ); + continue; + } + + $result .= $value[ $i ]; + } + + return $result; + } } diff --git a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php index 959d711df44af..016d4626a5a2a 100644 --- a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php +++ b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php @@ -177,4 +177,356 @@ public static function data_ident(): Generator { yield 'Semicolon' => array( 'a;b', 'a\\3B b' ); yield 'Backslash' => array( 'a\\b', 'a\\5C b' ); } + + /** + * Tests that serialize() writes an empty CSS comment between the tokens of a marked pair. + * + * @ticket 65738 + * @dataProvider data_marked_pairs + * + * @covers ::serialize + */ + public function test_marked_pair_is_separated_by_an_empty_comment( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_marked_pair_is_separated_by_an_empty_comment(). + * + * One case for each ✗ cell in the table in CSS Syntax Level 3 §9. + * + * @see https://www.w3.org/TR/css-syntax-3/#serialization + * + * @return Generator + */ + public static function data_marked_pairs(): Generator { + // Source text for the first token, keyed by its row heading. + $rows = array( + 'ident' => 'a', + 'at-keyword' => '@a', + 'hash' => '#a', + 'dimension' => '1px', + '#' => '#', + '-' => '-', + 'number' => '1', + '@' => '@', + '.' => '.', + '+' => '+', + '/' => '/', + ); + + // Source text and its serialization for the second token, keyed by its column heading. + $columns = array( + 'ident' => array( 'b', 'b' ), + 'function' => array( 'b()', 'b()' ), + 'url' => array( 'url(b)', 'url(b)' ), + 'bad url' => array( 'url(b c)', 'url(()' ), + '-' => array( '-', '-' ), + 'number' => array( '2', '2' ), + 'percentage' => array( '2%', '2%' ), + 'dimension' => array( '2px', '2px' ), + 'CDC' => array( '-->', '-->' ), + '(' => array( '()', '()' ), + '*' => array( '*', '*' ), + '%' => array( '%', '%' ), + ); + + $marked = array( + 'ident' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC', '(' ), + 'at-keyword' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'hash' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'dimension' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + '#' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + '-' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'number' => array( 'ident', 'function', 'url', 'bad url', 'number', 'percentage', 'dimension', 'CDC', '%' ), + '@' => array( 'ident', 'function', 'url', 'bad url', '-', 'CDC' ), + '.' => array( 'number', 'percentage', 'dimension' ), + '+' => array( 'number', 'percentage', 'dimension' ), + '/' => array( '*' ), + ); + + foreach ( $marked as $row => $row_columns ) { + foreach ( $row_columns as $column ) { + list( $column_css, $column_serialized ) = $columns[ $column ]; + yield "{$row} followed by {$column}" => array( + "{$rows[ $row ]}/**/{$column_css}", + "{$rows[ $row ]}/**/{$column_serialized}", + ); + } + } + } + + /** + * Tests that serialize() writes no comment between the tokens of an unmarked pair. + * + * @ticket 65738 + * @dataProvider data_unmarked_pairs + * + * @covers ::serialize + */ + public function test_unmarked_pair_is_not_separated( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_unmarked_pair_is_not_separated(). + * + * @return array[] + */ + public static function data_unmarked_pairs(): array { + return array( + 'ident followed by *' => array( 'a/**/*', 'a*' ), + 'ident followed by %' => array( 'a/**/%', 'a%' ), + 'ident followed by colon' => array( 'a/**/:', 'a:' ), + 'at-keyword followed by (' => array( '@a/**/()', '@a()' ), + 'hash followed by *' => array( '#a/**/*', '#a*' ), + 'dimension followed by %' => array( '1px/**/%', '1px%' ), + '# followed by %' => array( '#/**/%', '#%' ), + '- followed by *' => array( '-/**/*', '-*' ), + 'number followed by -' => array( '1/**/-', '1-' ), + 'number followed by (' => array( '1/**/()', '1()' ), + '@ followed by number' => array( '@/**/1', '@1' ), + '. followed by ident' => array( './**/a', '.a' ), + '+ followed by ident' => array( '+/**/a', '+a' ), + '/ followed by ident' => array( '//**/a', '/a' ), + 'string followed by ident' => array( '"a"/**/b', '"a"b' ), + 'url followed by ident' => array( 'url(a)/**/b', 'url(a)b' ), + 'function followed by ident' => array( 'a(/**/b)', 'a(b)' ), + ') followed by ident' => array( '(a)/**/b', '(a)b' ), + 'CDC followed by ident' => array( '-->/**/a', '-->a' ), + 'percentage followed by ident' => array( '1%/**/a', '1%a' ), + ); + } + + /** + * Tests that serialize() drops comments and writes each whitespace run as one space. + * + * @ticket 65738 + * @dataProvider data_whitespace_and_comments + * + * @covers ::serialize + */ + public function test_whitespace_and_comments( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + } + + /** + * Data provider for test_whitespace_and_comments(). + * + * @return array[] + */ + public static function data_whitespace_and_comments(): array { + return array( + 'empty input' => array( '', '' ), + 'one space' => array( 'a b', 'a b' ), + 'mixed whitespace run' => array( "a \t\r\n\f b", 'a b' ), + 'run broken by a comment' => array( 'a /* x */ b', 'a b' ), + 'whitespace then a comment' => array( 'a /**/b', 'a b' ), + 'a comment then whitespace' => array( 'a/**/ b', 'a b' ), + 'leading whitespace' => array( "\n a", ' a' ), + 'trailing whitespace' => array( "a \n", 'a ' ), + 'only whitespace' => array( "\t\n", ' ' ), + 'only a comment' => array( '/* x */', '' ), + 'comment in unmarked pair' => array( 'a/* x */,b', 'a,b' ), + 'unterminated comment' => array( 'a /* x', 'a ' ), + ); + } + + /** + * Tests how serialize() writes each token type. + * + * @ticket 65738 + * @dataProvider data_token_serialization + * + * @covers ::serialize + */ + public function test_token_serialization( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_token_serialization(). + * + * @return array[] + */ + public static function data_token_serialization(): array { + return array( + 'ident escapes decoded' => array( '\\66 oo', 'foo' ), + 'ident escapes added' => array( 'a\\:b', 'a\\3A b' ), + 'function' => array( '\\66 oo(1)', 'foo(1)' ), + 'at-keyword' => array( '@\\66 oo', '@foo' ), + 'hash id' => array( '#a\\:', '#a\\3A ' ), + 'hash id escaped digit' => array( '#\\31 a', '#\\31 a' ), + 'hash unrestricted' => array( '#1a', '#1a' ), + 'hash unrestricted escape' => array( '#1\\:', '#1\\3A ' ), + 'hash unrestricted hyphen' => array( '#-', '#-' ), + 'string' => array( "'a\"b'", '"a\\22 b"' ), + 'string ampersand' => array( '"a&b"', '"a\\26 b"' ), + 'string unterminated' => array( '"abc', '"abc"' ), + 'string escaped newline' => array( '"a\\A b"', '"a\\A b"' ), + 'string escaped carriage return' => array( '"a\\D b"', '"a\\D b"' ), + 'string escaped form feed' => array( '"a\\C b"', '"a\\C b"' ), + 'bad string' => array( "'a\nb", "\"\n b" ), + 'url' => array( 'url( a )', 'url(a)' ), + 'url empty' => array( 'url()', 'url()' ), + 'url escaped parenthesis' => array( 'url(a\\29 b)', 'url(a\\29 b)' ), + 'url escaped space' => array( 'url(a\\20 b)', 'url(a\\20 b)' ), + 'url escaped quote' => array( 'url(a\\"b)', 'url(a\\22 b)' ), + 'url escaped backslash' => array( 'url(a\\\\b)', 'url(a\\5C b)' ), + 'url escaped control' => array( 'url(a\\1 b)', 'url(a\\1 b)' ), + 'url escaped newline' => array( 'url(a\\A b)', 'url(a\\A b)' ), + 'url ampersand' => array( 'url(a&b)', 'url(a\\26 b)' ), + 'url NUL' => array( "url(a\x00b)", "url(a\u{FFFD}b)" ), + 'url unterminated' => array( 'url(a', 'url(a)' ), + 'url function with a string' => array( "url( 'a')", 'url( "a")' ), + 'bad url' => array( 'url(a b)', 'url(()' ), + 'number' => array( '+.5e-3', '+.5e-3' ), + 'percentage' => array( '-1.5%', '-1.5%' ), + 'dimension' => array( '1.5e3PX', '1.5e3PX' ), + 'dimension unit escape' => array( '1\\2D ', '1\\2D ' ), + 'dimension unit read as exponent' => array( '1\\65 3', '1\\65 3' ), + 'dimension unit read as signed exponent' => array( '1\\45 -3', '1\\45 -3' ), + 'dimension unit e' => array( '1e', '1e' ), + 'backslash delim keeps newline' => array( "a\\\nb", "a\\\n b" ), + 'delim' => array( '!', '!' ), + 'CDO and CDC' => array( '', '' ), + 'punctuation' => array( 'a:b;c,d', 'a:b;c,d' ), + ); + } + + /** + * Tests that serialize() closes blocks still open at the end, in reverse order. + * + * @ticket 65738 + * @dataProvider data_open_blocks + * + * @covers ::serialize + */ + public function test_open_blocks_are_closed( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + } + + /** + * Data provider for test_open_blocks_are_closed(). + * + * @return array[] + */ + public static function data_open_blocks(): array { + return array( + 'function' => array( 'calc(1px', 'calc(1px)' ), + 'nested blocks' => array( 'a(b[c{d', 'a(b[c{d}])' ), + 'closed block' => array( '(a)', '(a)' ), + 'mismatched closer' => array( '(]', '(])' ), + 'closer without a block' => array( ')', ')' ), + 'brace' => array( '{', '{}' ), + 'function with a string' => array( 'url("a"', 'url("a")' ), + 'inner block closed' => array( 'a((b)', 'a((b))' ), + ); + } + + /** + * Tests that the hash type flag is preserved. + * + * @ticket 65738 + * + * @covers ::serialize + */ + public function test_hash_flag_round_trips(): void { + foreach ( array( '#\\31 a', '#1a', '#a1', '#-1', '#--' ) as $css ) { + $tokens = self::tokenize( $css ); + $serialized = WP_CSS_Token_Serializer::serialize( $tokens ); + $reparsed = self::tokenize( $serialized ); + + $this->assertCount( 1, $reparsed, "{$css} must serialize as one token." ); + $this->assertSame( $tokens[0]->get_value(), $reparsed[0]->get_value(), "The value of {$css} changed." ); + $this->assertSame( $tokens[0]->get_flag(), $reparsed[0]->get_flag(), "The type flag of {$css} changed." ); + } + } + + /** + * Tests that serializing the corpus round-trips and is idempotent. + * + * The output must read as the input tokens, except that comments are dropped, + * whitespace runs are collapsed, and blocks still open are closed. + * + * @ticket 65738 + * @dataProvider data_corpus + * + * @covers ::serialize + */ + public function test_corpus_round_trips( string $css ): void { + $serialized = WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ); + + $expected = self::significant_tokens( $css ); + $actual = self::significant_tokens( $serialized ); + $closers = array_splice( $actual, count( $expected ) ); + + $this->assertSame( $expected, $actual, 'The output must read as the same tokens.' ); + foreach ( $closers as $closer ) { + $this->assertContains( + $closer[0], + array( WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, WP_CSS_Token_Processor::TOKEN_RIGHT_BRACKET, WP_CSS_Token_Processor::TOKEN_RIGHT_BRACE ), + 'Only block closers may follow the input tokens.' + ); + } + $this->assertSame( $serialized, WP_CSS_Token_Serializer::serialize( self::tokenize( $serialized ) ), 'Serialization must be idempotent.' ); + } + + /** + * Data provider for test_corpus_round_trips(). + * + * @return array[] + */ + public static function data_corpus(): array { + $cases = json_decode( file_get_contents( DIR_TESTDATA . '/css-api/css-test-cases.json' ), true ); + return array_map( + static function ( array $test_case ): array { + return array( $test_case['css'] ); + }, + $cases + ); + } + + /** + * Tokenizes CSS. + * + * @param string $css CSS text. + * @return WP_CSS_Token[] Tokens. + */ + private static function tokenize( string $css ): array { + $processor = WP_CSS_Token_Processor::create( $css ); + $tokens = array(); + while ( $processor->next_token() ) { + $tokens[] = $processor->get_token(); + } + return $tokens; + } + + /** + * Tokenizes CSS for comparison: comments are dropped, and each run of + * whitespace and comments that includes whitespace is one whitespace token. + * + * @param string $css CSS text. + * @return array[] Type, flag, value and unit of each token. + */ + private static function significant_tokens( string $css ): array { + $tokens = array(); + foreach ( self::tokenize( $css ) as $token ) { + $type = $token->get_type(); + if ( WP_CSS_Token_Processor::TOKEN_COMMENT === $type ) { + continue; + } + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE === $type ) { + if ( array( $type ) !== end( $tokens ) ) { + $tokens[] = array( $type ); + } + continue; + } + $tokens[] = array( $type, $token->get_flag(), $token->get_value(), $token->get_unit() ); + } + return $tokens; + } } From 34588608d023eb82abf16dbc14b4aa69c6414b61 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 20:34:10 +0400 Subject: [PATCH 16/77] Document why comment tokens are kept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CSS Syntax 3 §9 allows a tokenizer to preserve comments when they have no effect on parsing. The processor emits them so callers can find comments by byte offset. --- .../css-api/class-wp-css-token-processor.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/wp-includes/css-api/class-wp-css-token-processor.php b/src/wp-includes/css-api/class-wp-css-token-processor.php index 87c8b022c74c9..1d9e2ad7b4246 100644 --- a/src/wp-includes/css-api/class-wp-css-token-processor.php +++ b/src/wp-includes/css-api/class-wp-css-token-processor.php @@ -85,8 +85,21 @@ class WP_CSS_Token_Processor { * @see https://www.w3.org/TR/css-syntax-3/#tokenization */ public const TOKEN_WHITESPACE = 'whitespace-token'; - public const TOKEN_COMMENT = 'comment'; - public const TOKEN_STRING = 'string-token'; + + /** + * Comment tokens, such as `/* a *\/`. + * + * The specification's tokenizer produces no comment tokens. This processor + * does, so callers can find comments by their byte offsets. CSS Syntax + * Level 3 §9 allows this: + * + * > Implementations may preserve the contents of comments and their location in the token stream. + * > If they do, this preserved information must have no effect on the parsing step. + * + * @link https://www.w3.org/TR/css-syntax-3/#serialization + */ + public const TOKEN_COMMENT = 'comment'; + public const TOKEN_STRING = 'string-token'; /** * BAD-STRING tokens occur when a string contains an unescaped newline. From 47206d92f2f255db7a01c960e7f8e2a55ce2d973 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 21:06:49 +0400 Subject: [PATCH 17/77] Escape CR and FF in strings instead of normalizing them serialize_string() replaced CR, CRLF and FF in its argument with LF, as if the argument were CSS source. Its output is escaped and never preprocessed, so the replacement changed the value: a string token written from "a\D b" read back with a newline. CR and FF are now written as \D and \C, so the string reads back as the same value; NULL still becomes U+FFFD. A caller that wants normalized line endings normalizes before calling. serialize() writes string tokens with serialize_string() again, and the private quote_string() is removed. --- .../css-api/class-wp-css-token-serializer.php | 32 +++----------- .../tests/css-api/wpCssTokenSerializer.php | 43 +++++-------------- 2 files changed, 16 insertions(+), 59 deletions(-) diff --git a/src/wp-includes/css-api/class-wp-css-token-serializer.php b/src/wp-includes/css-api/class-wp-css-token-serializer.php index ac4f4d23dc870..a49c7dac836c3 100644 --- a/src/wp-includes/css-api/class-wp-css-token-serializer.php +++ b/src/wp-includes/css-api/class-wp-css-token-serializer.php @@ -307,7 +307,7 @@ private static function serialize_token( WP_CSS_Token $token ): string { : '#' . self::escape_name_code_points( $value ); case WP_CSS_Token_Processor::TOKEN_STRING: - return self::quote_string( $value ); + return self::serialize_string( $value ); case WP_CSS_Token_Processor::TOKEN_URL: return 'url(' . self::serialize_url_value( $value ) . ')'; @@ -431,6 +431,9 @@ public static function serialize_ident( string $value ): string { * or may be complicated for rudimentary CSS or HTML processors to handle * are encoded using Unicode escape sequences. * + * The string reads back as the same value, except that NULL becomes U+FFFD. + * Line endings are not normalized: CR and FF are escaped as `\D ` and `\C `. + * * @see https://www.w3.org/TR/css-syntax-3/#escaping * * @since 7.2.0 @@ -439,31 +442,6 @@ public static function serialize_ident( string $value ): string { * @return string Serialized quoted CSS string text. */ public static function serialize_string( string $value ): string { - // Normalize newlines. https://www.w3.org/TR/css-syntax-3/#input-preprocessing - $value = strtr( - $value, - array( - "\r\n" => "\n", - "\r" => "\n", - "\f" => "\n", - ) - ); - - return self::quote_string( $value ); - } - - /** - * Creates a quoted CSS string from a decoded value without normalizing newlines. - * - * A CR or FF in a token value comes from an escape such as `\D `, so it is - * escaped, not written as a newline. - * - * @since 7.2.0 - * - * @param string $value Decoded string value to encode as a quoted CSS string. - * @return string Serialized quoted CSS string text. - */ - private static function quote_string( string $value ): string { $value = wp_scrub_utf8( $value ); $escaped = strtr( $value, @@ -471,7 +449,7 @@ private static function quote_string( string $value ): string { // Escape existing backslashes to prevent unintentional escapes in result. '\\' => '\\5C ', - // Pre-processing replaces NULLs. Replace as necessary. + // CSS cannot represent NULL; preprocessing replaces it with U+FFFD. "\0" => "\u{FFFD}", // Newlines must be escaped in CSS strings. diff --git a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php index 016d4626a5a2a..1bc1bd758cee0 100644 --- a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php +++ b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php @@ -16,27 +16,14 @@ class Tests_CssApi_WpCssTokenSerializer extends WP_UnitTestCase { public function test_string_escaping( string $input, string $expected ): void { $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize_string( $input ) ); - /** - * Ensure that a single, equivalent CSS string is produced. - * - * CSS pre-processing normalization is applied to the input to ensure - * a match can be found. - * - * @see https://www.w3.org/TR/css-syntax-3/#input-preprocessing + /* + * Ensure that a single CSS string with the same value is produced. + * CSS cannot represent NULL, which becomes U+FFFD. */ $processor = WP_CSS_Token_Processor::create( WP_CSS_Token_Serializer::serialize_string( $input ) ); $processor->next_token(); $this->assertSame( WP_CSS_Token_Processor::TOKEN_STRING, $processor->get_token_type() ); - $expected_decoded_value = strtr( - $input, - array( - "\r\n" => "\n", - "\r" => "\n", - "\f" => "\n", - "\0" => '�', - ) - ); - $this->assertSame( $expected_decoded_value, $processor->get_token_value() ); + $this->assertSame( str_replace( "\0", '�', $input ), $processor->get_token_value() ); $this->assertFalse( $processor->next_token() ); } @@ -64,11 +51,11 @@ public static function data_string_escaping(): array { // NULL byte → U+FFFD replacement character. 'null byte' => array( "a\0b", '"a�b"' ), - // Newline normalization — all variants become \A escape. + // Newlines are escaped as they are; CR and FF are not normalized to LF. 'LF' => array( "a\nb", '"a\A b"' ), - 'CR' => array( "a\rb", '"a\A b"' ), - 'CRLF as single escape' => array( "a\r\nb", '"a\A b"' ), - 'form feed' => array( "a\fb", '"a\A b"' ), + 'CR' => array( "a\rb", '"a\D b"' ), + 'CRLF' => array( "a\r\nb", '"a\D \A b"' ), + 'form feed' => array( "a\fb", '"a\C b"' ), // HTML-problematic characters. 'HTML characters < > &' => array( 'ac&d', '"a\3C b\3E c\26 d"' ), @@ -99,7 +86,7 @@ public function test_docblock_example(): void { */ public function test_mixed_newlines(): void { $input = "line1\nline2\rline3\r\nline4\fline5"; - $expected = '"line1\A line2\A line3\A line4\A line5"'; + $expected = '"line1\A line2\D line3\D \A line4\C line5"'; $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize_string( $input ) ); } @@ -126,16 +113,7 @@ public function test_ident( string $input, string $expected ): void { $this->assertTrue( $processor->next_token() ); $this->assertSame( WP_CSS_Token_Processor::TOKEN_IDENT, $processor->get_token_type() ); - $expected_decoded_value = strtr( - $input, - array( - "\r\n" => "\n", - "\r" => "\n", - "\f" => "\n", - "\0" => '�', - ) - ); - $this->assertSame( $expected_decoded_value, $processor->get_token_value() ); + $this->assertSame( str_replace( "\0", '�', $input ), $processor->get_token_value() ); $this->assertFalse( $processor->next_token() ); } @@ -167,6 +145,7 @@ public static function data_ident(): Generator { yield 'Multiple spaces within' => array( 'a b c', 'a\\20 b\\20 c' ); yield 'Tab within' => array( "has\ttab", 'has\\9 tab' ); yield 'Newline within' => array( "has\nnewline", 'has\\A newline' ); + yield 'Carriage return within' => array( "has\rreturn", 'has\\D return' ); yield 'NULL byte within' => array( "has\0null", 'has�null' ); yield 'DEL within' => array( "has\x7Fdelete", 'has\\7F delete' ); From 2626801d0766b35eeefd4332d1865b65c5f772e8 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 21:07:32 +0400 Subject: [PATCH 18/77] Do not spell CDO or CDC from other tokens when serializing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CSS Syntax 3 §9 table compares token types in pairs. It misses two sequences that the tokenizer reads as a CDO or CDC token: `<`, `!` and a token that starts with `--` spell ``. So `` as `-->`. An empty comment is now written before the last token of each, so the output reads as the same tokens. --- .../css-api/class-wp-css-token-serializer.php | 33 ++++++++++++++--- .../tests/css-api/wpCssTokenSerializer.php | 37 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/wp-includes/css-api/class-wp-css-token-serializer.php b/src/wp-includes/css-api/class-wp-css-token-serializer.php index a49c7dac836c3..c4d4c780ba440 100644 --- a/src/wp-includes/css-api/class-wp-css-token-serializer.php +++ b/src/wp-includes/css-api/class-wp-css-token-serializer.php @@ -212,6 +212,12 @@ class WP_CSS_Token_Serializer { * Comments are not preserved, so an empty CSS comment is written between * the tokens of a marked pair when no whitespace is between them. * + * The table compares token types in pairs, so it misses two sequences that + * the tokenizer reads as a CDO or CDC token: `<`, `!` and a token that + * starts with `--`, which spell ``. An empty CSS comment is written before the last token + * of each. + * * Bad-string and bad-url tokens have no value. A bad string is written as * a quote followed by a newline, where the tokenizer ends one; a bad url is * written as `url(()`. @@ -243,6 +249,10 @@ public static function serialize( iterable $tokens ): string { // Pair key of the last token written, or the whitespace type after a space. $last = null; + // Text of the last two tokens written, or a space after whitespace. + $last_text = ''; + $before_last_text = ''; + foreach ( $tokens as $token ) { $type = $token->get_type(); @@ -252,19 +262,30 @@ public static function serialize( iterable $tokens ): string { if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE === $type ) { if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE !== $last ) { - $output .= ' '; - $last = WP_CSS_Token_Processor::TOKEN_WHITESPACE; + $output .= ' '; + $last = WP_CSS_Token_Processor::TOKEN_WHITESPACE; + $before_last_text = $last_text; + $last_text = ' '; } continue; } - $key = WP_CSS_Token_Processor::TOKEN_DELIM === $type ? $token->get_value() : $type; - if ( null !== $last && isset( self::MARKED_PAIRS[ $last ] ) && in_array( $key, self::MARKED_PAIRS[ $last ], true ) ) { + $key = WP_CSS_Token_Processor::TOKEN_DELIM === $type ? $token->get_value() : $type; + $text = self::serialize_token( $token ); + + if ( + ( null !== $last && isset( self::MARKED_PAIRS[ $last ] ) && in_array( $key, self::MARKED_PAIRS[ $last ], true ) ) || + // The table misses `<`, `!`, `--x`, which reads as ``. + ( '<' === $before_last_text && '!' === $last_text && 0 === strncmp( $text, '--', 2 ) ) || + ( '--' === $last_text && '>' === $text ) + ) { $output .= '/**/'; } - $last = $key; - $output .= self::serialize_token( $token ); + $output .= $text; + $last = $key; + $before_last_text = $last_text; + $last_text = $text; if ( isset( self::BLOCK_CLOSERS[ $type ] ) ) { $closers[] = self::BLOCK_CLOSERS[ $type ]; diff --git a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php index 1bc1bd758cee0..f4ce9281c9081 100644 --- a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php +++ b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php @@ -376,6 +376,43 @@ public static function data_token_serialization(): array { ); } + /** + * Tests that serialize() does not spell a CDO or CDC token from other tokens. + * + * The table in CSS Syntax Level 3 §9 compares token pairs by type. It does + * not cover `<`, `!` and a token that starts with `--`, which spell ``. + * + * @ticket 65738 + * @dataProvider data_cdo_and_cdc_sequences + * + * @covers ::serialize + */ + public function test_cdo_and_cdc_are_not_spelled( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_cdo_and_cdc_are_not_spelled(). + * + * @return array[] + */ + public static function data_cdo_and_cdc_sequences(): array { + return array( + '< ! ident' => array( ' array( ' array( ' array( '', '' ), + 'ident -- >' => array( '--/**/>', '--/**/>' ), + '< space ! ident' => array( '< !/**/--x', '< !--x' ), + '! ident without <' => array( 'a!/**/--x', 'a!--x' ), + '< ! ident without --' => array( '' => array( 'a--/**/>', 'a-->' ), + 'ident -- space >' => array( '-- >', '-- >' ), + ); + } + /** * Tests that serialize() closes blocks still open at the end, in reverse order. * From 08f3485b3bbae147b3894d57d074c6ee559e9423 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 22:07:54 +0400 Subject: [PATCH 19/77] Make the token serializer internal A token list can hold tokens that have no valid CSS form, so the public way to write CSS will be at the declaration level, where the style attribute processor exposes only valid declarations. WP_CSS_Token_Serializer is marked @access private; its methods stay public so other core classes can call them. serialize_ident() and serialize_string() had no caller outside the serializer. They become escape_ident(), public for property names, and the private escape_string(); url values use the private escape_url(), which now returns the whole url token. CSSOM defines "serialize an identifier" and "serialize a string" with a different escape set, so the new names avoid suggesting those algorithms. The string escaping tests now run through serialize(). The class documentation states what the output guarantees: it parses to the same CSS as the tokens. It is not always valid CSS: bad-string and bad-url tokens are written as the same bad tokens. --- .../css-api/class-wp-css-token-serializer.php | 86 +++++++------- .../tests/css-api/wpCssTokenSerializer.php | 105 +++++++----------- 2 files changed, 84 insertions(+), 107 deletions(-) diff --git a/src/wp-includes/css-api/class-wp-css-token-serializer.php b/src/wp-includes/css-api/class-wp-css-token-serializer.php index c4d4c780ba440..2c451c42d75aa 100644 --- a/src/wp-includes/css-api/class-wp-css-token-serializer.php +++ b/src/wp-includes/css-api/class-wp-css-token-serializer.php @@ -8,16 +8,16 @@ */ /** - * Serializes CSS tokens and values as CSS text. + * Writes tokens read by WP_CSS_Token_Processor as CSS text. * - * serialize() writes tokens read by WP_CSS_Token_Processor. serialize_ident() - * and serialize_string() write a plain PHP string as an identifier or as a - * quoted string. + * This class is not a public API. Its methods are public so other core + * classes can call them. * - * Output is valid CSS that reads as the same tokens. Which characters are + * The output parses to the same CSS as the tokens. Which characters are * escaped may change. * * @since 7.2.0 + * @access private * * @see WP_CSS_Token_Processor */ @@ -147,7 +147,7 @@ class WP_CSS_Token_Serializer { ); /** - * Serializes tokens as CSS text that reads as the same tokens. + * Serializes tokens as CSS text that parses to the same CSS. * * This follows CSS Syntax Level 3 §9 Serialization: * @@ -220,7 +220,9 @@ class WP_CSS_Token_Serializer { * * Bad-string and bad-url tokens have no value. A bad string is written as * a quote followed by a newline, where the tokenizer ends one; a bad url is - * written as `url(()`. + * written as `url(()`. Both re-tokenize as the same bad token. If a later + * step replaces the newline after a bad string with a space, the quote + * starts a string that includes the text after it. * * Strings and urls cut off by the end of the input are written closed. * Blocks still open at the end are closed in reverse order, so `a(b[c` is @@ -314,24 +316,24 @@ private static function serialize_token( WP_CSS_Token $token ): string { switch ( $token->get_type() ) { case WP_CSS_Token_Processor::TOKEN_IDENT: - return self::serialize_ident( $value ); + return self::escape_ident( $value ); case WP_CSS_Token_Processor::TOKEN_FUNCTION: - return self::serialize_ident( $value ) . '('; + return self::escape_ident( $value ) . '('; case WP_CSS_Token_Processor::TOKEN_AT_KEYWORD: - return '@' . self::serialize_ident( $value ); + return '@' . self::escape_ident( $value ); case WP_CSS_Token_Processor::TOKEN_HASH: return WP_CSS_Token_Processor::HASH_TOKEN_ID === $token->get_flag() - ? '#' . self::serialize_ident( $value ) - : '#' . self::escape_name_code_points( $value ); + ? '#' . self::escape_ident( $value ) + : '#' . self::escape_name( $value ); case WP_CSS_Token_Processor::TOKEN_STRING: - return self::serialize_string( $value ); + return self::escape_string( $value ); case WP_CSS_Token_Processor::TOKEN_URL: - return 'url(' . self::serialize_url_value( $value ) . ')'; + return self::escape_url( $value ); case WP_CSS_Token_Processor::TOKEN_NUMBER: return $value; @@ -340,7 +342,7 @@ private static function serialize_token( WP_CSS_Token $token ): string { return $value . '%'; case WP_CSS_Token_Processor::TOKEN_DIMENSION: - $unit = self::serialize_ident( $token->get_unit() ); + $unit = self::escape_ident( $token->get_unit() ); // A unit that starts with `e`, an optional `-`, and a digit would read as an exponent. if ( 1 === preg_match( '/^[eE]-?[0-9]/', $unit ) ) { $unit = sprintf( '\\%X ', ord( $unit[0] ) ) . substr( $unit, 1 ); @@ -356,25 +358,24 @@ private static function serialize_token( WP_CSS_Token $token ): string { } /** - * Serializes a plain PHP string as CSS identifier text. + * Escapes a decoded value as identifier text. * - * Characters not valid in CSS identifiers are hex-escaped. This uses - * the same safety escaping as {@see WP_CSS_Token_Serializer::serialize_string()} for HTML - * and CSS-sensitive characters, plus escaping of whitespace and other - * characters not permitted in idents. + * Characters not valid in CSS identifiers are hex-escaped, as are the + * characters escape_string() escapes for HTML and CSS. * - * An empty string serializes to an empty string. Callers that require an - * ident token must reject an empty value. + * An empty value returns an empty string. Callers that require an ident + * token must reject an empty value. * * @see https://www.w3.org/TR/css-syntax-3/#escaping * @see https://www.w3.org/TR/css-syntax-3/#would-start-an-identifier * * @since 7.2.0 + * @access private * - * @param string $value Decoded string value to encode as a CSS identifier. - * @return string Serialized CSS identifier text. + * @param string $value Decoded value. + * @return string Identifier text. */ - public static function serialize_ident( string $value ): string { + public static function escape_ident( string $value ): string { $value = wp_scrub_utf8( $value ); $result = ''; $length = strlen( $value ); @@ -441,16 +442,11 @@ public static function serialize_ident( string $value ): string { } /** - * Creates a quoted CSS string from a plain PHP string value. + * Escapes a decoded value as a quoted string. * - * Example: - * $value = 'CSS & a ""; - * - * CSS strings are quoted, and many characters that are problematic in HTML - * or may be complicated for rudimentary CSS or HTML processors to handle - * are encoded using Unicode escape sequences. + * Many characters that are problematic in HTML or may be complicated for + * rudimentary CSS or HTML processors to handle are encoded using Unicode + * escape sequences. * * The string reads back as the same value, except that NULL becomes U+FFFD. * Line endings are not normalized: CR and FF are escaped as `\D ` and `\C `. @@ -459,10 +455,10 @@ public static function serialize_ident( string $value ): string { * * @since 7.2.0 * - * @param string $value Decoded string value to encode as a quoted CSS string. - * @return string Serialized quoted CSS string text. + * @param string $value Decoded value. + * @return string Quoted string text. */ - public static function serialize_string( string $value ): string { + private static function escape_string( string $value ): string { $value = wp_scrub_utf8( $value ); $escaped = strtr( $value, @@ -502,7 +498,7 @@ public static function serialize_string( string $value ): string { * * Letters, digits, `-`, `_` and code points U+0080 and above are written * as they are; every other code point is hex-escaped. Unlike - * serialize_ident(), a leading digit or `-` is not escaped, as the name of + * escape_ident(), a leading digit or `-` is not escaped, as the name of * a hash token with the "unrestricted" flag requires. * * @since 7.2.0 @@ -510,7 +506,7 @@ public static function serialize_string( string $value ): string { * @param string $value Decoded name. * @return string Escaped name. */ - private static function escape_name_code_points( string $value ): string { + private static function escape_name( string $value ): string { $value = wp_scrub_utf8( $value ); $result = ''; $length = strlen( $value ); @@ -542,20 +538,20 @@ private static function escape_name_code_points( string $value ): string { } /** - * Escapes the value of a url token. + * Escapes a decoded value as an unquoted url. * * The code points an unquoted url cannot contain are hex-escaped: quotes, * parentheses, `\`, whitespace and non-printable code points. So are the - * code points serialize_string() escapes for HTML and CSS: `< > & , ; { }`. + * code points escape_string() escapes for HTML and CSS: `< > & , ; { }`. * * @see https://www.w3.org/TR/css-syntax-3/#consume-url-token * * @since 7.2.0 * - * @param string $value Decoded url value. - * @return string Escaped url value, without `url(` and `)`. + * @param string $value Decoded value. + * @return string Url text, including `url(` and `)`. */ - private static function serialize_url_value( string $value ): string { + private static function escape_url( string $value ): string { $value = wp_scrub_utf8( $value ); $result = ''; $length = strlen( $value ); @@ -580,6 +576,6 @@ private static function serialize_url_value( string $value ): string { $result .= $value[ $i ]; } - return $result; + return "url({$result})"; } } diff --git a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php index f4ce9281c9081..084d339f407f3 100644 --- a/tests/phpunit/tests/css-api/wpCssTokenSerializer.php +++ b/tests/phpunit/tests/css-api/wpCssTokenSerializer.php @@ -9,99 +9,84 @@ */ class Tests_CssApi_WpCssTokenSerializer extends WP_UnitTestCase { /** + * Tests that serialize() escapes a string token again from its decoded value. + * + * @ticket 65738 * @dataProvider data_string_escaping * - * @covers ::serialize_string + * @covers ::serialize */ - public function test_string_escaping( string $input, string $expected ): void { - $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize_string( $input ) ); - - /* - * Ensure that a single CSS string with the same value is produced. - * CSS cannot represent NULL, which becomes U+FFFD. - */ - $processor = WP_CSS_Token_Processor::create( WP_CSS_Token_Serializer::serialize_string( $input ) ); - $processor->next_token(); - $this->assertSame( WP_CSS_Token_Processor::TOKEN_STRING, $processor->get_token_type() ); - $this->assertSame( str_replace( "\0", '�', $input ), $processor->get_token_value() ); - $this->assertFalse( $processor->next_token() ); + public function test_string_escaping( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); } /** - * Data provider for string escaping cases. + * Data provider for test_string_escaping(). * - * @return array + * @return array[] */ public static function data_string_escaping(): array { return array( // Passthrough — no escaping needed, only quoting. - 'empty string' => array( '', '""' ), - 'simple ASCII' => array( 'Arial', '"Arial"' ), - 'spaces preserved' => array( 'Exo 2', '"Exo 2"' ), - 'leading/trailing spaces' => array( ' Arial ', '" Arial "' ), - 'whitespace-only' => array( ' ', '" "' ), - 'numbers pass through' => array( '12345', '"12345"' ), - 'non-ASCII passthrough' => array( 'café', '"café"' ), + 'empty string' => array( "''", '""' ), + 'simple ASCII' => array( "'Arial'", '"Arial"' ), + 'spaces preserved' => array( "'Exo 2'", '"Exo 2"' ), + 'leading/trailing spaces' => array( "' Arial '", '" Arial "' ), + 'whitespace-only' => array( "' '", '" "' ), + 'numbers pass through' => array( "'12345'", '"12345"' ), + 'non-ASCII passthrough' => array( "'café'", '"café"' ), // Backslash escaping — must happen first to prevent double-escaping. - 'backslash' => array( 'Back\\Slash', '"Back\5C Slash"' ), - 'double backslash' => array( '\\\\', '"\5C \5C "' ), - 'backslash before quote' => array( "a\\'b", '"a\5C \27 b"' ), + 'backslash' => array( '"Back\\\\Slash"', '"Back\\5C Slash"' ), + 'double backslash' => array( '"\\\\\\\\"', '"\\5C \\5C "' ), + 'backslash before quote' => array( '"a\\\\\'b"', '"a\\5C \\27 b"' ), - // NULL byte → U+FFFD replacement character. - 'null byte' => array( "a\0b", '"a�b"' ), + // NULL → U+FFFD replacement character. + 'null byte' => array( "\"a\x00b\"", '"a�b"' ), + 'zero escape' => array( '"a\\0 b"', '"a�b"' ), - // Newlines are escaped as they are; CR and FF are not normalized to LF. - 'LF' => array( "a\nb", '"a\A b"' ), - 'CR' => array( "a\rb", '"a\D b"' ), - 'CRLF' => array( "a\r\nb", '"a\D \A b"' ), - 'form feed' => array( "a\fb", '"a\C b"' ), + // Escaped newlines keep their code points; CR and FF are not normalized to LF. + 'LF' => array( '"a\\A b"', '"a\\A b"' ), + 'CR' => array( '"a\\D b"', '"a\\D b"' ), + 'CRLF' => array( '"a\\D \\A b"', '"a\\D \\A b"' ), + 'form feed' => array( '"a\\C b"', '"a\\C b"' ), // HTML-problematic characters. - 'HTML characters < > &' => array( 'ac&d', '"a\3C b\3E c\26 d"' ), + 'HTML characters < > &' => array( '"ac&d"', '"a\\3C b\\3E c\\26 d"' ), // CSS-problematic characters. - 'CSS syntax , ; { }' => array( 'a,b;c{d}', '"a\2C b\3B c\7B d\7D "' ), - 'single quote' => array( "CSS's strings", '"CSS\27 s strings"' ), - 'double quote' => array( 'Say "Hi"', '"Say \22 Hi\22 "' ), + 'CSS syntax , ; { }' => array( '"a,b;c{d}"', '"a\\2C b\\3B c\\7B d\\7D "' ), + 'single quote' => array( '"CSS\'s strings"', '"CSS\\27 s strings"' ), + 'double quote' => array( '\'Say "Hi"\'', '"Say \\22 Hi\\22 "' ), ); } /** - * Tests the example from the class docblock. + * Tests the example in the serialize() docblock. * - * @covers ::serialize_string - */ - public function test_docblock_example(): void { - $value = 'CSS & a " + * * * Modern CSS no longer needs these, but they're preserved for compatibility. * In stylesheets, they're typically treated like whitespace. * - * @see https://www.w3.org/TR/css-syntax-3/#typedef-CDC-token + * @see WP_CSS_Token_Processor::TOKEN_CDO + * @link https://www.w3.org/TR/css-syntax-3/#typedef-cdc-token */ public const TOKEN_CDC = 'CDC-token'; /** - * CDO (Comment Delimiter Open) token: ) + * If followed by another hyphen and `>`, this is a CDC token (`-->`). * * Comment Delimiter Close - legacy HTML comment syntax in CSS. * - * @see https://www.w3.org/TR/css-syntax-3/#CDC-token-diagram + * @link https://www.w3.org/TR/css-syntax-3/#CDC-token-diagram */ if ( $this->at + 2 < $this->length && @@ -570,11 +617,11 @@ public function next_token(): bool { /* * U+003C LESS-THAN SIGN (<) - * If followed by !--, this is a CDO token (", "startIndex": 0, "endIndex": 3, - "normalized": "-->", "value": null }, { @@ -2751,7 +2471,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2764,7 +2483,6 @@ "raw": "foo", "startIndex": 0, "endIndex": 3, - "normalized": "foo", "value": "foo" }, { @@ -2772,7 +2490,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2785,7 +2502,6 @@ "raw": "--", "startIndex": 0, "endIndex": 2, - "normalized": "--", "value": "--" }, { @@ -2793,7 +2509,6 @@ "raw": "\n", "startIndex": 2, "endIndex": 3, - "normalized": "\n", "value": null } ] @@ -2806,7 +2521,6 @@ "raw": "--0", "startIndex": 0, "endIndex": 3, - "normalized": "--0", "value": "--0" }, { @@ -2814,7 +2528,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2827,7 +2540,6 @@ "raw": "-", "startIndex": 0, "endIndex": 1, - "normalized": "-", "value": "-" }, { @@ -2835,7 +2547,6 @@ "raw": "\\", "startIndex": 1, "endIndex": 2, - "normalized": "\\", "value": "\\" }, { @@ -2843,7 +2554,6 @@ "raw": "\n", "startIndex": 2, "endIndex": 3, - "normalized": "\n", "value": null } ] @@ -2856,7 +2566,6 @@ "raw": "-\\ ", "startIndex": 0, "endIndex": 3, - "normalized": "- ", "value": "- " }, { @@ -2864,7 +2573,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2877,7 +2585,6 @@ "raw": "--💅", "startIndex": 0, "endIndex": 6, - "normalized": "--💅", "value": "--💅" }, { @@ -2885,7 +2592,6 @@ "raw": "\n", "startIndex": 6, "endIndex": 7, - "normalized": "\n", "value": null } ] @@ -2898,7 +2604,6 @@ "raw": "-§", "startIndex": 0, "endIndex": 3, - "normalized": "-§", "value": "-§" }, { @@ -2906,7 +2611,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2919,7 +2623,6 @@ "raw": "-×", "startIndex": 0, "endIndex": 3, - "normalized": "-×", "value": "-×" }, { @@ -2927,7 +2630,6 @@ "raw": "\n", "startIndex": 3, "endIndex": 4, - "normalized": "\n", "value": null } ] @@ -2940,7 +2642,6 @@ "raw": "--a𐀀", "startIndex": 0, "endIndex": 7, - "normalized": "--a𐀀", "value": "--a𐀀" }, { @@ -2948,7 +2649,6 @@ "raw": "\n", "startIndex": 7, "endIndex": 8, - "normalized": "\n", "value": null } ] @@ -2961,7 +2661,6 @@ "raw": "url(foo)", "startIndex": 0, "endIndex": 8, - "normalized": "url(foo)", "value": "foo" }, { @@ -2969,7 +2668,6 @@ "raw": "\n", "startIndex": 8, "endIndex": 9, - "normalized": "\n", "value": null } ] @@ -2982,7 +2680,6 @@ "raw": "\\75 Rl(foo)", "startIndex": 0, "endIndex": 11, - "normalized": "uRl(foo)", "value": "foo" }, { @@ -2990,7 +2687,6 @@ "raw": "\n", "startIndex": 11, "endIndex": 12, - "normalized": "\n", "value": null } ] @@ -3003,7 +2699,6 @@ "raw": "uR\\6c (foo)", "startIndex": 0, "endIndex": 11, - "normalized": "uRl(foo)", "value": "foo" }, { @@ -3011,7 +2706,6 @@ "raw": "\n", "startIndex": 11, "endIndex": 12, - "normalized": "\n", "value": null } ] @@ -3024,7 +2718,6 @@ "raw": "url(", "startIndex": 0, "endIndex": 4, - "normalized": "url(", "value": "url" }, { @@ -3032,7 +2725,6 @@ "raw": "'foo'", "startIndex": 4, "endIndex": 9, - "normalized": "'foo'", "value": "foo" }, { @@ -3040,7 +2732,6 @@ "raw": ")", "startIndex": 9, "endIndex": 10, - "normalized": ")", "value": null }, { @@ -3048,7 +2739,6 @@ "raw": "\n", "startIndex": 10, "endIndex": 11, - "normalized": "\n", "value": null } ] @@ -3061,7 +2751,6 @@ "raw": "url(", "startIndex": 0, "endIndex": 4, - "normalized": "url(", "value": "url" }, { @@ -3069,7 +2758,6 @@ "raw": " ", "startIndex": 4, "endIndex": 5, - "normalized": " ", "value": null }, { @@ -3077,7 +2765,6 @@ "raw": "'foo'", "startIndex": 5, "endIndex": 10, - "normalized": "'foo'", "value": "foo" }, { @@ -3085,7 +2772,6 @@ "raw": ")", "startIndex": 10, "endIndex": 11, - "normalized": ")", "value": null }, { @@ -3093,7 +2779,6 @@ "raw": "\n", "startIndex": 11, "endIndex": 12, - "normalized": "\n", "value": null } ] @@ -3106,7 +2791,6 @@ "raw": "url(", "startIndex": 0, "endIndex": 4, - "normalized": "url(", "value": "url" }, { @@ -3114,7 +2798,6 @@ "raw": " ", "startIndex": 4, "endIndex": 6, - "normalized": " ", "value": null }, { @@ -3122,7 +2805,6 @@ "raw": "'foo'", "startIndex": 6, "endIndex": 11, - "normalized": "'foo'", "value": "foo" }, { @@ -3130,7 +2812,6 @@ "raw": ")", "startIndex": 11, "endIndex": 12, - "normalized": ")", "value": null }, { @@ -3138,7 +2819,6 @@ "raw": "\n", "startIndex": 12, "endIndex": 13, - "normalized": "\n", "value": null } ] @@ -3151,7 +2831,6 @@ "raw": "url(", "startIndex": 0, "endIndex": 4, - "normalized": "url(", "value": "url" }, { @@ -3159,7 +2838,6 @@ "raw": " ", "startIndex": 4, "endIndex": 7, - "normalized": " ", "value": null }, { @@ -3167,7 +2845,6 @@ "raw": "'foo'", "startIndex": 7, "endIndex": 12, - "normalized": "'foo'", "value": "foo" }, { @@ -3175,7 +2852,6 @@ "raw": ")", "startIndex": 12, "endIndex": 13, - "normalized": ")", "value": null }, { @@ -3183,7 +2859,6 @@ "raw": "\n", "startIndex": 13, "endIndex": 14, - "normalized": "\n", "value": null } ] @@ -3196,7 +2871,6 @@ "raw": "not-url(", "startIndex": 0, "endIndex": 8, - "normalized": "not-url(", "value": "not-url" }, { @@ -3204,7 +2878,6 @@ "raw": " ", "startIndex": 8, "endIndex": 11, - "normalized": " ", "value": null }, { @@ -3212,7 +2885,6 @@ "raw": "'foo'", "startIndex": 11, "endIndex": 16, - "normalized": "'foo'", "value": "foo" }, { @@ -3220,7 +2892,6 @@ "raw": ")", "startIndex": 16, "endIndex": 17, - "normalized": ")", "value": null }, { @@ -3228,7 +2899,6 @@ "raw": "\n", "startIndex": 17, "endIndex": 18, - "normalized": "\n", "value": null } ] @@ -3241,7 +2911,6 @@ "raw": "url( foo)", "startIndex": 0, "endIndex": 11, - "normalized": "url( foo)", "value": "foo" }, { @@ -3249,7 +2918,6 @@ "raw": "\n", "startIndex": 11, "endIndex": 12, - "normalized": "\n", "value": null } ] @@ -3262,7 +2930,6 @@ "raw": "{", "startIndex": 0, "endIndex": 1, - "normalized": "{", "value": null }, { @@ -3270,7 +2937,6 @@ "raw": "\n", "startIndex": 1, "endIndex": 2, - "normalized": "\n", "value": null } ] @@ -3283,7 +2949,6 @@ "raw": "(", "startIndex": 0, "endIndex": 1, - "normalized": "(", "value": null }, { @@ -3291,7 +2956,6 @@ "raw": "\n", "startIndex": 1, "endIndex": 2, - "normalized": "\n", "value": null } ] @@ -3304,7 +2968,6 @@ "raw": "[", "startIndex": 0, "endIndex": 1, - "normalized": "[", "value": null }, { @@ -3312,7 +2975,6 @@ "raw": "\n", "startIndex": 1, "endIndex": 2, - "normalized": "\n", "value": null } ] @@ -3325,7 +2987,6 @@ "raw": "<", "startIndex": 0, "endIndex": 1, - "normalized": "<", "value": "<" }, { @@ -3333,7 +2994,6 @@ "raw": "\n", "startIndex": 1, "endIndex": 2, - "normalized": "\n", "value": null } ] @@ -3346,7 +3006,6 @@ "raw": "