Skip to content

Commit

Permalink
Re-enable APIDiff tests (#8196)
Browse files Browse the repository at this point in the history 
Re-enable APIDiff tests.

### Motivation:

I noticed that APIDiff tests are skipped when using most recent Swift
(confirmed on 6.0.2 on MacOS, 6.0.3 on Linux and 6.2-dev nightly on
Linux). Upon enabling I noticed that 2 tests are failing.

### Modifications:

- Enable the test suite by modifying condition from implicit versioning
to explicit versioning
 - Update the test suite to be run by default
- Adjust asserts to be less brittle for `testFilters` and
`testCheckVendedModulesOnly`

### Result:

 - Tests are run by default
- All tests pass (verified on 6.0.2 on MacOS, 6.0.3 on Linux and
6.2-dev-2024-12-22-a nightly on Linux)
  • Loading branch information
@yyvch
yyvch authored Jan 9, 2025
1 parent 86b0ff1 commit ba273eb
Showing 1 changed file with 31 additions and 41 deletions.
72 changes: 31 additions & 41 deletions Tests/CommandsTests/APIDiffTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,25 +39,27 @@ final class APIDiffTests: CommandsTestCase {

func skipIfApiDigesterUnsupportedOrUnset() throws {
try skipIfApiDigesterUnsupported()
// The following is added to separate out the integration point testing of the API
// diff digester with SwiftPM from the functionality tests of the digester itself
guard Environment.current["SWIFTPM_TEST_API_DIFF_OUTPUT"] == "1" else {
throw XCTSkip("Env var SWIFTPM_TEST_API_DIFF_OUTPUT must be set to test the output")
}
// Opt out from testing the API diff if necessary.
// TODO: Cleanup after March 2025.
// The opt-in/opt-out mechanism doesn't seem to be used.
// It is kept around for abundance of caution. If "why is it needed"
// not identified by March 2025, then it is OK to remove it.
try XCTSkipIf(
Environment.current["SWIFTPM_TEST_API_DIFF_OUTPUT"] == "0",
"Env var SWIFTPM_TEST_API_DIFF_OUTPUT is set to skip the API diff tests."
)
}

func skipIfApiDigesterUnsupported() throws {
// swift-api-digester is required to run tests.
guard (try? UserToolchain.default.getSwiftAPIDigester()) != nil else {
throw XCTSkip("swift-api-digester unavailable")
}
// SwiftPM's swift-api-digester integration relies on post-5.5 bugfixes and features,
// not all of which can be tested for easily. Fortunately, we can test for the
// `-disable-fail-on-error` option, and any version which supports this flag
// will meet the other requirements.
guard DriverSupport.checkSupportedFrontendFlags(flags: ["disable-fail-on-error"], toolchain: try UserToolchain.default, fileSystem: localFileSystem) else {
throw XCTSkip("swift-api-digester is too old")
}
// The tests rely on swift-api-digester post-5.5 version and are certain
// to work with Swift compiler v6.0 and later.
#if compiler(<6.0)
throw XCTSkip("Skipping because test requires at least Swift compiler v6.0")
#endif
}

func testInvokeAPIDiffDigester() async throws {
Expand Down Expand Up @@ -165,18 +167,13 @@ final class APIDiffTests: CommandsTestCase {
string: "public class Qux<T, U> { private let x = 1 }"
)
await XCTAssertThrowsCommandExecutionError(try await execute(["diagnose-api-breaking-changes", "1.2.3"], packagePath: packageRoot)) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertMatch(error.stdout, .contains("2 breaking changes detected in Bar"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: import Baz has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Foo"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Bar"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Baz"))

// Qux is not part of a library product, so any API changes should be ignored
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}
}
}
Expand Down Expand Up @@ -204,33 +201,26 @@ final class APIDiffTests: CommandsTestCase {
await XCTAssertThrowsCommandExecutionError(
try await execute(["diagnose-api-breaking-changes", "1.2.3", "--products", "One", "--targets", "Bar"], packagePath: packageRoot)
) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertMatch(error.stdout, .contains("2 breaking changes detected in Bar"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: import Baz has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Foo"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Bar"))

XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
// Baz and Qux are not included in the filter, so any API changes should be ignored.
XCTAssertNoMatch(error.stdout, .contains("Baz"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}

// Diff a target which didn't have a baseline generated as part of the first invocation
await XCTAssertThrowsCommandExecutionError(
try await execute(["diagnose-api-breaking-changes", "1.2.3", "--targets", "Baz"], packagePath: packageRoot)
) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Baz"))

XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Bar"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
// Only Baz is included, we should not see any other API changes.
XCTAssertNoMatch(error.stdout, .contains("Foo"))
XCTAssertNoMatch(error.stdout, .contains("Bar"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}

// Test diagnostics
Expand Down

0 comments on commit ba273eb

Please sign in to comment.