Skip to content

Commit 75e5a97

Browse files
rnrornro
authored
Switch allocation benchmarks to GHA (#488)
### Motivation: This will allow us to decommission the old CI hooks. ### Modifications: * Migrate the existing tests to use the package-benchmark framework. * Delete the obsolete integration tests * Remove allocations test references from the docker files. ### Result: Allocation counts are checked using GitHub Actions.
1 parent 61cfaf9 commit 75e5a97

26 files changed

+239
-272
lines changed

‎.github/workflows/pull_request.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,13 @@ jobs:
2525
uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
2626
with:
2727
name: "Integration test"
28-
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation"
28+
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh"
29+
30+
benchmarks:
31+
name: Benchmarks
32+
uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
33+
with:
34+
benchmark_package_path: "Benchmarks"
2935

3036
cxx-interop:
3137
name: Cxx interop

‎.github/workflows/scheduled.yml

+6
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,9 @@ jobs:
2222
with:
2323
name: "Integration test"
2424
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation"
25+
26+
benchmarks:
27+
name: Benchmarks
28+
uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
29+
with:
30+
benchmark_package_path: "Benchmarks"

‎Benchmarks/Benchmarks/NIOSSLBenchmarks/Benchmarks.swift

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the SwiftNIO open source project
4+
//
5+
// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
6+
// Licensed under Apache License v2.0
7+
//
8+
// See LICENSE.txt for license information
9+
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
10+
//
11+
// SPDX-License-Identifier: Apache-2.0
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
import Benchmark
16+
import NIOCore
17+
import NIOEmbedded
18+
import NIOSSL
19+
20+
let benchmarks = {
21+
let defaultMetrics: [BenchmarkMetric] = [
22+
.mallocCountTotal
23+
]
24+
25+
Benchmark(
26+
"SimpleHandshake",
27+
configuration: .init(
28+
metrics: defaultMetrics,
29+
scalingFactor: .kilo,
30+
maxDuration: .seconds(10_000_000),
31+
maxIterations: 10,
32+
thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])]
33+
)
34+
) { benchmark in
35+
try runSimpleHandshake(
36+
handshakeCount: benchmark.scaledIterations.upperBound
37+
)
38+
}
39+
40+
Benchmark(
41+
"ManyWrites",
42+
configuration: .init(
43+
metrics: defaultMetrics,
44+
scalingFactor: .kilo,
45+
maxDuration: .seconds(10_000_000),
46+
maxIterations: 10,
47+
thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])]
48+
)
49+
) { benchmark in
50+
try runManyWrites(
51+
writeCount: benchmark.scaledIterations.upperBound
52+
)
53+
}
54+
}

‎Benchmarks/Benchmarks/NIOSSLBenchmarks/ManyWrites.swift

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the SwiftNIO open source project
4+
//
5+
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
6+
// Licensed under Apache License v2.0
7+
//
8+
// See LICENSE.txt for license information
9+
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
10+
//
11+
// SPDX-License-Identifier: Apache-2.0
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
import NIOCore
16+
import NIOEmbedded
17+
import NIOSSL
18+
19+
func runManyWrites(writeCount: Int) throws {
20+
let serverContext = try NIOSSLContext(
21+
configuration: .makeServerConfiguration(
22+
certificateChain: [.certificate(.forTesting())],
23+
privateKey: .privateKey(.forTesting())
24+
)
25+
)
26+
27+
var clientConfig = TLSConfiguration.makeClientConfiguration()
28+
clientConfig.trustRoots = try .certificates([.forTesting()])
29+
let clientContext = try NIOSSLContext(configuration: clientConfig)
30+
31+
let dummyAddress = try SocketAddress(ipAddress: "1.2.3.4", port: 5678)
32+
let backToBack = BackToBackEmbeddedChannel()
33+
let serverHandler = NIOSSLServerHandler(context: serverContext)
34+
let clientHandler = try NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
35+
try backToBack.client.pipeline.addHandler(clientHandler).wait()
36+
try backToBack.server.pipeline.addHandler(serverHandler).wait()
37+
38+
// To trigger activation of both channels we use connect().
39+
try backToBack.client.connect(to: dummyAddress).wait()
40+
try backToBack.server.connect(to: dummyAddress).wait()
41+
42+
try backToBack.interactInMemory()
43+
44+
// Let's try 512 bytes.
45+
var buffer = backToBack.client.allocator.buffer(capacity: 512)
46+
buffer.writeBytes(repeatElement(0, count: 512))
47+
48+
for _ in 0..<writeCount {
49+
// A vector of 100 writes.
50+
for _ in 0..<100 {
51+
backToBack.client.write(buffer, promise: nil)
52+
}
53+
backToBack.client.flush()
54+
55+
try backToBack.interactInMemory()
56+
57+
// Pull any data out of the server to avoid ballooning in memory.
58+
while let _ = try backToBack.server.readInbound(as: ByteBuffer.self) {}
59+
}
60+
}

‎IntegrationTests/tests_02_allocation_counters/test_01_resources/shared.swift ‎Benchmarks/Benchmarks/NIOSSLBenchmarks/Shared.swift

File renamed without changes.

‎Benchmarks/Benchmarks/NIOSSLBenchmarks/SimpleHandshake.swift

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the SwiftNIO open source project
4+
//
5+
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
6+
// Licensed under Apache License v2.0
7+
//
8+
// See LICENSE.txt for license information
9+
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
10+
//
11+
// SPDX-License-Identifier: Apache-2.0
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
import NIOCore
16+
import NIOEmbedded
17+
import NIOSSL
18+
19+
func runSimpleHandshake(handshakeCount: Int) throws {
20+
let serverContext = try NIOSSLContext(
21+
configuration: .makeServerConfiguration(
22+
certificateChain: [.certificate(.forTesting())],
23+
privateKey: .privateKey(.forTesting())
24+
)
25+
)
26+
27+
var clientConfig = TLSConfiguration.makeClientConfiguration()
28+
clientConfig.trustRoots = try .certificates([.forTesting()])
29+
let clientContext = try NIOSSLContext(configuration: clientConfig)
30+
31+
let dummyAddress = try SocketAddress(ipAddress: "1.2.3.4", port: 5678)
32+
33+
for _ in 0..<handshakeCount {
34+
let backToBack = BackToBackEmbeddedChannel()
35+
let serverHandler = NIOSSLServerHandler(context: serverContext)
36+
let clientHandler = try NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
37+
try backToBack.client.pipeline.addHandler(clientHandler).wait()
38+
try backToBack.server.pipeline.addHandler(serverHandler).wait()
39+
40+
// To trigger activation of both channels we use connect().
41+
try backToBack.client.connect(to: dummyAddress).wait()
42+
try backToBack.server.connect(to: dummyAddress).wait()
43+
44+
try backToBack.interactInMemory()
45+
46+
// Ok, now do shutdown.
47+
backToBack.client.close(promise: nil)
48+
try backToBack.interactInMemory()
49+
try backToBack.client.closeFuture.wait()
50+
try backToBack.server.closeFuture.wait()
51+
}
52+
}

‎Benchmarks/Package.swift

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// swift-tools-version: 5.9
2+
3+
import PackageDescription
4+
5+
let package = Package(
6+
name: "benchmarks",
7+
platforms: [
8+
.macOS("14")
9+
],
10+
dependencies: [
11+
.package(path: "../"),
12+
.package(url: "https://github.com/apple/swift-nio.git", from: "2.54.0"),
13+
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.22.0"),
14+
],
15+
targets: [
16+
.executableTarget(
17+
name: "NIOSSHBenchmarks",
18+
dependencies: [
19+
.product(name: "Benchmark", package: "package-benchmark"),
20+
.product(name: "NIOSSL", package: "swift-nio-ssl"),
21+
.product(name: "NIOCore", package: "swift-nio"),
22+
.product(name: "NIOEmbedded", package: "swift-nio"),
23+
],
24+
path: "Benchmarks/NIOSSLBenchmarks",
25+
plugins: [
26+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
27+
]
28+
)
29+
]
30+
)

‎Benchmarks/Thresholds/5.10/NIOSSHBenchmarks.ManyWrites.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 201948
3+
}

‎Benchmarks/Thresholds/5.10/NIOSSHBenchmarks.SimpleHandshake.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 644865
3+
}

‎Benchmarks/Thresholds/5.9/NIOSSHBenchmarks.ManyWrites.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 201948
3+
}

‎Benchmarks/Thresholds/5.9/NIOSSHBenchmarks.SimpleHandshake.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 644868
3+
}

‎Benchmarks/Thresholds/6.0/NIOSSHBenchmarks.ManyWrites.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 201947
3+
}

‎Benchmarks/Thresholds/6.0/NIOSSHBenchmarks.SimpleHandshake.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 643873
3+
}

‎Benchmarks/Thresholds/nightly-6.0/NIOSSHBenchmarks.ManyWrites.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 201947
3+
}

‎Benchmarks/Thresholds/nightly-6.0/NIOSSHBenchmarks.SimpleHandshake.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 643873
3+
}

‎Benchmarks/Thresholds/nightly-main/NIOSSHBenchmarks.ManyWrites.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 201947
3+
}

‎Benchmarks/Thresholds/nightly-main/NIOSSHBenchmarks.SimpleHandshake.p90.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 643873
3+
}

‎IntegrationTests/tests_02_allocation_counters/defines.sh

-14
This file was deleted.

‎IntegrationTests/tests_02_allocation_counters/test_01_allocation_counts.sh

-54
This file was deleted.

0 commit comments

Comments
 (0)