Add JSON benchmarks and make benchmark package build for more targets (#810)

* Add JSON benchmarks and make benchmark package build for more targets

* Address review feedback

Author: parkera

Benchmarks/Benchmarks/Calendar/BenchmarkCalendar.swift

@@ -13,18 +13,18 @@
 import Benchmark
 import func Benchmark.blackHole
 
-#if FOUNDATION_FRAMEWORK
-import Foundation
-#else
+#if os(macOS) && USE_PACKAGE
 import FoundationEssentials
 import FoundationInternationalization
+#else
+import Foundation
 #endif
 
 let benchmarks = {
     Benchmark.defaultConfiguration.maxIterations = 1_000
     Benchmark.defaultConfiguration.maxDuration = .seconds(3)
     Benchmark.defaultConfiguration.scalingFactor = .kilo
-    Benchmark.defaultConfiguration.metrics = [.cpuTotal, .wallClock, .mallocCountTotal, .throughput]
+    Benchmark.defaultConfiguration.metrics = [.cpuTotal, .mallocCountTotal, .throughput]
 
     let thanksgivingComponents = DateComponents(month: 11, weekday: 5, weekdayOrdinal: 4)
     let cal = Calendar(identifier: .gregorian)
@@ -42,6 +42,7 @@ let benchmarks = {
             }
         }
     }
+
     Benchmark("nextThousandThanksgivings") { benchmark in
         var count = 1000
         cal.enumerateDates(startingAfter: thanksgivingStart, matching: thanksgivingComponents, matchingPolicy: .nextTime) { result, exactMatch, stop in
@@ -51,26 +52,34 @@ let benchmarks = {
             }
         }
     }
-    Benchmark("nextThousandThanksgivingsSequence") { benchmark in
-        var count = 1000
-        for _ in cal.dates(byMatching: thanksgivingComponents, startingAt: thanksgivingStart, matchingPolicy: .nextTime) {
-            count -= 1
-            if count == 0 {
-                break
+
+    // Only available in Swift 6 for non-Darwin platforms, macOS 15 for Darwin
+    #if swift(>=6.0)
+    if #available(macOS 15, *) {
+        Benchmark("nextThousandThanksgivingsSequence") { benchmark in
+            var count = 1000
+            for _ in cal.dates(byMatching: thanksgivingComponents, startingAt: thanksgivingStart, matchingPolicy: .nextTime) {
+                count -= 1
+                if count == 0 {
+                    break
+                }
             }
         }
-    }
-    Benchmark("nextThousandThanksgivingsUsingRecurrenceRule") { benchmark in
-        var rule = Calendar.RecurrenceRule(calendar: cal, frequency: .yearly, end: .afterOccurrences(1000))
-        rule.months = [11]
-        rule.weekdays = [.nth(4, .thursday)]
-        rule.matchingPolicy = .nextTime
-        var count = 0
-        for _ in rule.recurrences(of: thanksgivingStart) {
-            count += 1
+
+        Benchmark("nextThousandThanksgivingsUsingRecurrenceRule") { benchmark in
+            var rule = Calendar.RecurrenceRule(calendar: cal, frequency: .yearly, end: .afterOccurrences(1000))
+            rule.months = [11]
+            rule.weekdays = [.nth(4, .thursday)]
+            rule.matchingPolicy = .nextTime
+            var count = 0
+            for _ in rule.recurrences(of: thanksgivingStart) {
+                count += 1
+            }
+            assert(count == 1000)
         }
-        assert(count == 1000)
-    }
+    } // #available(macOS 15, *)
+    #endif // swift(>=6.0)
+
     Benchmark("CurrentDateComponentsFromThanksgivings") { benchmark in
         var count = 1000
         currentCalendar.enumerateDates(startingAfter: thanksgivingStart, matching: thanksgivingComponents, matchingPolicy: .nextTime) { result, exactMatch, stop in
@@ -82,9 +91,17 @@ let benchmarks = {
         }
     }
 
+    // MARK: - Allocations
+
     let reference = Date(timeIntervalSinceReferenceDate: 496359355.795410) //2016-09-23T14:35:55-0700
 
-    Benchmark("allocationsForFixedCalendars", configuration: .init(scalingFactor: .mega)) { benchmark in
+    let allocationsConfiguration = Benchmark.Configuration(
+        metrics: [.cpuTotal, .mallocCountTotal, .peakMemoryResident, .throughput],
+        timeUnits: .nanoseconds,
+        scalingFactor: .mega
+    )
+
+    Benchmark("allocationsForFixedCalendars", configuration: allocationsConfiguration) { benchmark in
         for _ in benchmark.scaledIterations {
             // Fixed calendar
             let cal = Calendar(identifier: .gregorian)
@@ -93,7 +110,7 @@ let benchmarks = {
         }
     }
 
-    Benchmark("allocationsForCurrentCalendar", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("allocationsForCurrentCalendar", configuration: allocationsConfiguration) { benchmark in
         for _ in benchmark.scaledIterations {
             // Current calendar
             let cal = Calendar.current
@@ -102,7 +119,7 @@ let benchmarks = {
         }
     }
 
-    Benchmark("allocationsForAutoupdatingCurrentCalendar", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("allocationsForAutoupdatingCurrentCalendar", configuration: allocationsConfiguration) { benchmark in
         for _ in benchmark.scaledIterations {
             // Autoupdating current calendar
             let cal = Calendar.autoupdatingCurrent
@@ -111,23 +128,23 @@ let benchmarks = {
         }
     }
 
-    Benchmark("copyOnWritePerformance", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("copyOnWritePerformance", configuration: allocationsConfiguration) { benchmark in
         var cal = Calendar(identifier: .gregorian)
         for i in benchmark.scaledIterations {
             cal.firstWeekday = i % 2
             assert(cal.firstWeekday == i % 2)
         }
     }
 
-    Benchmark("copyOnWritePerformanceNoDiff", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("copyOnWritePerformanceNoDiff", configuration: allocationsConfiguration) { benchmark in
         var cal = Calendar(identifier: .gregorian)
         let tz = TimeZone(secondsFromGMT: 1800)!
         for _ in benchmark.scaledIterations {
             cal.timeZone = tz
         }
     }
 
-    Benchmark("allocationsForFixedLocale", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("allocationsForFixedLocale", configuration: allocationsConfiguration) { benchmark in
         // Fixed locale
         for _ in benchmark.scaledIterations {
             let loc = Locale(identifier: "en_US")
@@ -136,7 +153,7 @@ let benchmarks = {
         }
     }
 
-    Benchmark("allocationsForCurrentLocale", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("allocationsForCurrentLocale", configuration: allocationsConfiguration) { benchmark in
         // Current locale
         for _ in benchmark.scaledIterations {
             let loc = Locale.current
@@ -145,15 +162,17 @@ let benchmarks = {
         }
     }
 
-    Benchmark("allocationsForAutoupdatingCurrentLocale", configuration: .init(scalingFactor: .mega)) { benchmark in
+    Benchmark("allocationsForAutoupdatingCurrentLocale", configuration: allocationsConfiguration) { benchmark in
         // Autoupdating current locale
         for _ in benchmark.scaledIterations {
             let loc = Locale.autoupdatingCurrent
             let identifier = loc.identifier
             assert(identifier == "en_US")
         }
     }
-        
+
+    // MARK: - Identifiers
+
     Benchmark("identifierFromComponents", configuration: .init(scalingFactor: .mega)) { benchmark in
         let c1 = ["kCFLocaleLanguageCodeKey" : "en"]
         let c2 = ["kCFLocaleLanguageCodeKey" : "zh",

Benchmarks/Benchmarks/DataIO/BenchmarkDataIO.swift

@@ -13,12 +13,15 @@
 import Benchmark
 import func Benchmark.blackHole
 
-#if FOUNDATION_FRAMEWORK
-import Foundation
+#if os(macOS) && USE_PACKAGE
+import FoundationEssentials
 #else
-@testable import FoundationEssentials
+import Foundation
 #endif
 
+#if !FOUNDATION_FRAMEWORK
+private func autoreleasepool<T>(_ block: () -> T) -> T { block() }
+#endif
 
 #if canImport(Glibc)
 import Glibc
@@ -27,16 +30,13 @@ import Glibc
 import Darwin
 #endif
 
-#if !FOUNDATION_FRAMEWORK
-func testPath() -> String {
-    // Generate a random file name
-    FileManager.default.temporaryDirectory.path.appendingPathComponent("testfile-\(UUID().uuidString)")
-}
-#else
 func testPath() -> URL {
+    #if compiler(>=6)
     FileManager.default.temporaryDirectory.appending(path: "testfile-\(UUID().uuidString)", directoryHint: .notDirectory)
+    #else
+    FileManager.default.temporaryDirectory.appendingPathComponent("testfile-\(UUID().uuidString)")
+    #endif
 }
-#endif
 
 func generateTestData(count: Int) -> Data {
     let memory = malloc(count)!
@@ -51,17 +51,10 @@ func generateTestData(count: Int) -> Data {
     return Data(bytesNoCopy: ptr, count: count, deallocator: .free)
 }
 
-#if !FOUNDATION_FRAMEWORK
-func cleanup(at path: String) {
-    try? FileManager.default.removeItem(atPath: path)
-    // Ignore any errors
-}
-#else
 func cleanup(at path: URL) {
     try? FileManager.default.removeItem(at: path)
     // Ignore any errors
 }
-#endif
 
 // 16 MB file, big enough to trigger things like chunking
 let data = generateTestData(count: 1 << 24)
@@ -74,7 +67,13 @@ let benchmarks = {
     Benchmark.defaultConfiguration.maxIterations = 1_000_000_000
     Benchmark.defaultConfiguration.maxDuration = .seconds(3)
     Benchmark.defaultConfiguration.scalingFactor = .kilo
+    #if os(macOS)
+    Benchmark.defaultConfiguration.metrics = [.cpuTotal, .wallClock, .mallocCountTotal, .throughput, .syscalls]
+    #elseif os(Linux)
+    Benchmark.defaultConfiguration.metrics = [.cpuTotal, .wallClock, .mallocCountTotal, .throughput, .readSyscalls, .writeSyscalls]
+    #else
     Benchmark.defaultConfiguration.metrics = [.cpuTotal, .wallClock, .mallocCountTotal, .throughput]
+    #endif
 
     Benchmark("read-write-emptyFile") { benchmark in
         let path = testPath()
@@ -118,7 +117,10 @@ let benchmarks = {
 
     // MARK: base64
 
-    Benchmark("base64-encode", configuration: .init(scalingFactor: .kilo)) { benchmark in
+    Benchmark("base64-encode", configuration: .init(
+        metrics: [.cpuTotal, .mallocCountTotal, .peakMemoryResident, .throughput],
+        scalingFactor: .kilo)
+    ) { benchmark in
         for _ in benchmark.scaledIterations {
             autoreleasepool {
                 blackHole(base64Data.base64EncodedString())
@@ -127,7 +129,10 @@ let benchmarks = {
     }
 
 
-    Benchmark("base64-decode", configuration: .init(scalingFactor: .kilo)) { benchmark in
+    Benchmark("base64-decode", configuration: .init(
+        metrics: [.cpuTotal, .mallocCountTotal, .peakMemoryResident, .throughput],
+        scalingFactor: .kilo)
+    ) { benchmark in
         for _ in benchmark.scaledIterations {
             autoreleasepool {
                 blackHole(Data(base64Encoded: base64DataString))

Benchmarks/Benchmarks/Essentials/BenchmarkEssentials.swift

@@ -13,10 +13,10 @@
 import Benchmark
 import func Benchmark.blackHole
 
-#if FOUNDATION_FRAMEWORK
-import Foundation
-#else
+#if os(macOS) && USE_PACKAGE
 import FoundationEssentials
+#else
+import Foundation
 #endif
 
 let benchmarks = {

Benchmarks/Benchmarks/Formatting/BenchmarkFormatting.swift

@@ -13,10 +13,10 @@
 import Benchmark
 import func Benchmark.blackHole
 
-#if FOUNDATION_FRAMEWORK
-import Foundation
-#else
+#if os(macOS) && USE_PACKAGE
 import FoundationEssentials
+#else
+import Foundation
 #endif
 
 let benchmarks = {
@@ -27,6 +27,9 @@ let benchmarks = {
 
     let date = Date(timeIntervalSinceReferenceDate: 665076946.0)
 
+    // ISO8601FormatStyle is only available in Swift 6 or newer, macOS 12 or newer
+    #if compiler(>=6)
+
     let iso8601 = Date.ISO8601FormatStyle()
     let formats: [Date.ISO8601FormatStyle] = [
         iso8601.year().month().day().dateSeparator(.dash),
@@ -57,4 +60,6 @@ let benchmarks = {
             }
         }
     }
+
+    #endif // swift(>=6)
 }

Benchmarks/Benchmarks/JSON/Canada.swift

@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// Swift port of [Native-JSON Benchmark](https://github.com/miloyip/nativejson-benchmark)
+/*
+The MIT License (MIT)
+
+Copyright (c) 2014 Milo Yip
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+enum ObjType : String, Codable {
+    case featureCollection = "FeatureCollection"
+    case feature = "Feature"
+    case polygon = "Polygon"
+}
+
+struct Feature : Codable {
+    var type: ObjType
+    var properties: [String: String]
+    var geometry: Geometry
+}
+
+struct Geometry : Codable {
+    struct Coordinate : Codable {
+        var latitude: Double
+        var longitude: Double
+
+        init(from decoder: any Decoder) throws {
+            var container = try decoder.unkeyedContainer()
+            latitude = try container.decode(Double.self)
+            longitude = try container.decode(Double.self)
+        }
+        
+        func encode(to encoder: any Encoder) throws {
+            var container = encoder.unkeyedContainer()
+            try container.encode(latitude)
+            try container.encode(longitude)
+        }
+    }
+
+    var type: ObjType
+    var coordinates: [[Coordinate]]
+}
+
+struct FeatureCollection : Codable {
+    var type: ObjType
+    var features: [Feature]
+}
+