Merge pull request #25957 from Catfish-Man/someone-set-up-us-the-bom-5.1

Bridge non-ASCII SmallStrings as native Swift Strings rather than by creating CFStrings. This gets consistent behavior with non-smol Strings when the String contains a BOM

Author: Catfish-Man

stdlib/public/core/StringBridge.swift

@@ -321,15 +321,25 @@ extension String {
   @_effects(releasenone)
   public // SPI(Foundation)
   func _bridgeToObjectiveCImpl() -> AnyObject {
-    if _guts.isSmall {
+    // Smol ASCII a) may bridge to tagged pointers, b) can't contain a BOM
+    if _guts.isSmallASCII {
       return _guts.asSmall.withUTF8 { bufPtr in
         return _createCFString(
-            bufPtr.baseAddress._unsafelyUnwrappedUnchecked,
-            bufPtr.count,
-            kCFStringEncodingUTF8
+          bufPtr.baseAddress._unsafelyUnwrappedUnchecked,
+          bufPtr.count,
+          kCFStringEncodingUTF8
         )
       }
     }
+    if _guts.isSmall {
+        // We can't form a tagged pointer String, so grow to a non-small String,
+        // and bridge that instead. Also avoids CF deleting any BOM that may be
+        // present
+        var copy = self
+        copy._guts.grow(_SmallString.capacity + 1)
+        _internalInvariant(!copy._guts.isSmall)
+        return copy._bridgeToObjectiveCImpl()
+    }
     if _guts._object.isImmortal {
       // TODO: We'd rather emit a valid ObjC object statically than create a
       // shared string class instance.

test/stdlib/TestNSString.swift

@@ -30,16 +30,40 @@ import FoundationBridgeObjC
 
 class TestNSString : TestNSStringSuper {
 
-    func test_equalOverflow() {
-      let cyrillic = "чебурашка@ящик-с-апельсинами.рф"
-      let other = getNSStringEqualTestString()
-      print(NSStringBridgeTestEqual(cyrillic, other))
-    }
+  func test_equalOverflow() {
+    let cyrillic = "чебурашка@ящик-с-апельсинами.рф"
+    let other = getNSStringEqualTestString()
+    print(NSStringBridgeTestEqual(cyrillic, other))
+  }
+  
+  func test_smallString_BOM() {
+    let bom = "\u{FEFF}" // U+FEFF (ZERO WIDTH NO-BREAK SPACE)
+//    expectEqual(1, NSString(string: bom).length)
+//    expectEqual(4, NSString(string: "\(bom)abc").length)
+//    expectEqual(5, NSString(string: "\(bom)\(bom)abc").length)
+//    expectEqual(4, NSString(string: "a\(bom)bc").length)
+//    expectEqual(13, NSString(string: "\(bom)234567890123").length)
+//    expectEqual(14, NSString(string: "\(bom)2345678901234").length)
+    
+    expectEqual(1, (bom as NSString).length)
+    expectEqual(4, ("\(bom)abc" as NSString).length)
+    expectEqual(5, ("\(bom)\(bom)abc" as NSString).length)
+    expectEqual(4, ("a\(bom)bc" as NSString).length)
+    expectEqual(13, ("\(bom)234567890123" as NSString).length)
+    expectEqual(14, ("\(bom)2345678901234" as NSString).length)
+    
+    let string = "\(bom)abc"
+    let middleIndex = string.index(string.startIndex, offsetBy: 2)
+    string.enumerateSubstrings(in: middleIndex..<string.endIndex, options: .byLines) { (_, _, _, _) in }  //shouldn't crash
+  }
 
 }
 
 #if !FOUNDATION_XCTEST
 var NSStringTests = TestSuite("TestNSString")
 NSStringTests.test("test_equalOverflow") { TestNSString().test_equalOverflow() }
+NSStringTests.test("test_smallString_BOM") {
+  TestNSString().test_smallString_BOM()
+}
 runAllTests()
 #endif