Handle unprocessable whitespace-related unicode characters

Author: TTOzzi

Sources/SwiftFormat/PrettyPrint/WhitespaceLinter.swift

@@ -312,13 +312,14 @@ public class WhitespaceLinter {
     formattedRun: ArraySlice<UTF8.CodeUnit>
   ) {
     guard userRun != formattedRun else { return }
-
+    let userString = String(decoding: userRun, as: UTF8.self)
+    let formattedString = String(decoding: formattedRun, as: UTF8.self)
     // This assumes tabs will always be forbidden for inter-token spacing (but not for leading
     // indentation).
     if userRun.contains(utf8Tab) {
       diagnose(.spacingCharError, category: .spacingCharacter, utf8Offset: userIndex)
-    } else if formattedRun.count != userRun.count {
-      let delta = formattedRun.count - userRun.count
+    } else if formattedString.count != userString.count {
+      let delta = formattedString.count - userString.count
       diagnose(.spacingError(delta), category: .spacing, utf8Offset: userIndex)
     }
   }
@@ -339,20 +340,26 @@ public class WhitespaceLinter {
     startingAt offset: Int,
     in data: [UTF8.CodeUnit]
   ) -> ArraySlice<UTF8.CodeUnit> {
-    func isWhitespace(_ char: UTF8.CodeUnit) -> Bool {
-      switch char {
-      case UInt8(ascii: " "), UInt8(ascii: "\n"), UInt8(ascii: "\t"), UInt8(ascii: "\r"), /*VT*/ 0x0B, /*FF*/ 0x0C:
-        return true
+    var currentIndex = offset
+    while currentIndex < data.count {
+      if let unicodeException = UnicodeWhitespace.allCases.first(where: { exception in
+        let bytes = exception.utf8Bytes
+        return currentIndex + bytes.count <= data.count
+          && data[currentIndex..<currentIndex + bytes.count].elementsEqual(bytes)
+      }) {
+        currentIndex += unicodeException.utf8Bytes.count
+        continue
+      }
+
+      switch data[currentIndex] {
+      case UInt8(ascii: " "), UInt8(ascii: "\n"), UInt8(ascii: "\t"), UInt8(ascii: "\r"),
+        /*VT*/ 0x0B, /*FF*/ 0x0C:
+        currentIndex += 1
       default:
-        return false
+        return data[offset..<currentIndex]
       }
     }
-    guard
-      let whitespaceEnd = data[offset...].firstIndex(where: { !isWhitespace($0) })
-    else {
-      return data[offset..<data.endIndex]
-    }
-    return data[offset..<whitespaceEnd]
+    return data[offset..<currentIndex]
   }
 
   /// Returns the code unit at the given index, or nil if the index is the end of the data.
@@ -412,6 +419,22 @@ public class WhitespaceLinter {
   }
 }
 
+/// A collection of Unicode code points that represent non-standard whitespace.
+private enum UnicodeWhitespace: CaseIterable {
+  case u2028  // U+2028 LINE SEPARATOR
+  case u2029  // U+2029 PARAGRAPH SEPARATOR
+
+  /// Returns the UTF-8 byte sequence corresponding to the Unicode exception.
+  var utf8Bytes: [UTF8.CodeUnit] {
+    switch self {
+    case .u2028:
+      return [0xE2, 0x80, 0xA8]
+    case .u2029:
+      return [0xE2, 0x80, 0xA9]
+    }
+  }
+}
+
 /// Describes the composition of the whitespace that creates an indentation for a line of code.
 public enum WhitespaceIndentation: Equatable {
   /// The line has no preceding whitespace, meaning there's no indentation.

Tests/SwiftFormatTests/PrettyPrint/WhitespaceLintTests.swift

@@ -255,4 +255,27 @@ final class WhitespaceLintTests: WhitespaceTestCase {
       ]
     )
   }
+
+  func testUnexpectedUnicodeCharacters() {
+    assertWhitespaceLint(
+      input: """
+        // Hello World1️⃣\u{2028}
+        // Hello2️⃣\u{20}\u{2028}World
+        // Hello World3️⃣\u{2028}\u{2029}\u{2029}
+        // Hello World4️⃣\u{20}\u{20}\u{20}\u{2028}
+        """,
+      expected: """
+        // Hello World
+        // Hello World
+        // Hello World
+        // Hello World
+        """,
+      findings: [
+        FindingSpec("1️⃣", message: "remove trailing whitespace"),
+        FindingSpec("2️⃣", message: "remove 1 space"),
+        FindingSpec("3️⃣", message: "remove trailing whitespace"),
+        FindingSpec("4️⃣", message: "remove 4 spaces"),
+      ]
+    )
+  }
 }