Add support for a prebuilt swift-syntax library for macros

Package.swift

@@ -655,6 +655,14 @@ let package = Package(
             name: "swift-package-registry",
             dependencies: ["Commands", "PackageRegistryCommand"]
         ),
+        .executableTarget(
+            /** Utility to produce the artifacts for prebuilts */
+            name: "swift-build-prebuilts",
+            dependencies: [
+                "Basics",
+                "Workspace",
+            ]
+        ),
 
         // MARK: Support for Swift macros, should eventually move to a plugin-based solution
 

Sources/Basics/Archiver/ZipArchiver.swift

@@ -48,7 +48,9 @@ public struct ZipArchiver: Archiver, Cancellable {
             }
 
             #if os(Windows)
-            let process = AsyncProcess(arguments: ["tar.exe", "xf", archivePath.pathString, "-C", destinationPath.pathString])
+            // FileManager lost the ability to detect tar.exe as executable.
+            // It's part of system32 anyway so use the absolute path.
+            let process = AsyncProcess(arguments: ["C:\\Windows\\system32\\tar.exe", "xf", archivePath.pathString, "-C", destinationPath.pathString])
             #else
             let process = AsyncProcess(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString])
             #endif

Sources/Basics/Collections/IdentifiableSet.swift

@@ -122,3 +122,9 @@ extension IdentifiableSet: Hashable {
         }
     }
 }
+
+extension IdentifiableSet: ExpressibleByArrayLiteral {
+    public init(arrayLiteral elements: Element...) {
+        self.init(elements)
+    }
+}

Sources/Commands/CommandWorkspaceDelegate.swift

@@ -182,6 +182,42 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate {
         self.progressHandler(step, total, "Downloading \(artifacts)")
     }
 
+    /// The workspace has started downloading a binary artifact.
+    func willDownloadPrebuilt(from url: String, fromCache: Bool) {
+        if fromCache {
+            self.outputHandler("Fetching package prebuilt \(url) from cache", false)
+        } else {
+            self.outputHandler("Downloading package prebuilt \(url)", false)
+        }
+    }
+
+    /// The workspace has finished downloading a binary artifact.
+    func didDownloadPrebuilt(
+        from url: String,
+        result: Result<(path: AbsolutePath, fromCache: Bool), Error>,
+        duration: DispatchTimeInterval
+    ) {
+        guard case .success(let fetchDetails) = result, !self.observabilityScope.errorsReported else {
+            return
+        }
+
+        if fetchDetails.fromCache {
+            self.outputHandler("Fetched \(url) from cache (\(duration.descriptionInSeconds))", false)
+        } else {
+            self.outputHandler("Downloaded \(url) (\(duration.descriptionInSeconds))", false)
+        }
+    }
+
+    /// The workspace is downloading a binary artifact.
+    func downloadingPrebuilt(from url: String, bytesDownloaded: Int64, totalBytesToDownload: Int64?) {
+
+    }
+
+    /// The workspace finished downloading all binary artifacts.
+    func didDownloadAllPrebuilts() {
+
+    }
+
     // registry signature handlers
 
     func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version, completion: (Bool) -> Void) {

Sources/CoreCommands/Options.swift

@@ -186,6 +186,12 @@ public struct CachingOptions: ParsableArguments {
             self.init(rawValue: argument)
         }
     }
+
+    /// Whether to use macro prebuilts or not
+    @Flag(name: .customLong("experimental-prebuilts"),
+          inversion: .prefixedEnableDisable,
+          help: "Whether to use prebuilt swift-syntax libraries for macros")
+    public var usePrebuilts: Bool = false
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -80,6 +80,7 @@ public typealias WorkspaceDelegateProvider = (
     _ progressHandler: @escaping (Int64, Int64, String?) -> Void,
     _ inputHandler: @escaping (String, (String?) -> Void) -> Void
 ) -> WorkspaceDelegate
+
 public typealias WorkspaceLoaderProvider = (_ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope)
     -> WorkspaceLoader
 
@@ -470,7 +471,8 @@ public final class SwiftCommandState {
                     // TODO: should supportsAvailability be a flag as well?
                     .init(url: $0, supportsAvailability: true)
                 },
-                manifestImportRestrictions: .none
+                manifestImportRestrictions: .none,
+                usePrebuilts: options.caching.usePrebuilts
             ),
             cancellator: self.cancellator,
             initializationWarningHandler: { self.observabilityScope.emit(warning: $0) },

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -29,6 +29,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         customPlatformsRegistry: PlatformRegistry? = .none,
@@ -47,6 +48,7 @@ extension ModulesGraph {
             requiredDependencies: requiredDependencies,
             unsafeAllowedPackages: unsafeAllowedPackages,
             binaryArtifacts: binaryArtifacts,
+            prebuilts: prebuilts,
             shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
             createREPLProduct: createREPLProduct,
             traitConfiguration: nil,
@@ -69,6 +71,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]], // Product name to library mapping
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         traitConfiguration: TraitConfiguration? = nil,
@@ -212,7 +215,8 @@ extension ModulesGraph {
                     productFilter: node.productFilter,
                     path: packagePath,
                     additionalFileRules: additionalFileRules,
-                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],                
+                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],
+                    prebuilts: prebuilts,
                     shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
                     testEntryPointPath: testEntryPointPath,
                     createREPLProduct: manifest.packageKind.isRoot ? createREPLProduct : false,
@@ -246,6 +250,7 @@ extension ModulesGraph {
             manifestToPackage: manifestToPackage,
             rootManifests: root.manifests,
             unsafeAllowedPackages: unsafeAllowedPackages,
+            prebuilts: prebuilts,
             platformRegistry: customPlatformsRegistry ?? .default,
             platformVersionProvider: platformVersionProvider,
             fileSystem: fileSystem,
@@ -377,6 +382,7 @@ private func createResolvedPackages(
     // FIXME: This shouldn't be needed once <rdar://problem/33693433> is fixed.
     rootManifests: [PackageIdentity: Manifest],
     unsafeAllowedPackages: Set<PackageReference>,
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
     platformRegistry: PlatformRegistry,
     platformVersionProvider: PlatformVersionProvider,
     fileSystem: FileSystem,
@@ -665,6 +671,11 @@ private func createResolvedPackages(
                     }
                 }
 
+                if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
+                    // using a prebuilt instead.
+                    continue
+                }
+
                 // Find the product in this package's dependency products.
                 // Look it up by ID if module aliasing is used, otherwise by name.
                 let product = lookupByProductIDs ? productDependencyMap[productRef.identity] : productDependencyMap[productRef.name]

Sources/PackageGraph/ModulesGraph.swift

@@ -419,6 +419,7 @@ public func loadModulesGraph(
     fileSystem: FileSystem,
     manifests: [Manifest],
     binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]] = [:],
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]] = [:],
     explicitProduct: String? = .none,
     shouldCreateMultipleTestProducts: Bool = false,
     createREPLProduct: Bool = false,
@@ -451,6 +452,7 @@ public func loadModulesGraph(
             .swiftpmFileTypes,
         externalManifests: externalManifests,
         binaryArtifacts: binaryArtifacts,
+        prebuilts: prebuilts,
         shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
         createREPLProduct: createREPLProduct,
         customXCTestMinimumDeploymentTargets: customXCTestMinimumDeploymentTargets,

Sources/PackageLoading/PackageBuilder.swift

@@ -268,6 +268,32 @@ public struct BinaryArtifact {
     }
 }
 
+/// A structure representing a prebuilt library to be used instead of a source dependency
+public struct PrebuiltLibrary {
+    /// The package reference.
+    public let packageRef: PackageReference
+
+    /// The name of the binary target the artifact corresponds to.
+    public let libraryName: String
+
+    /// The path to the extracted prebuilt artifacts
+    public let path: AbsolutePath
+
+    /// The products in the library
+    public let products: [String]
+
+    /// The C modules that need their includes directory added to the include path
+    public let cModules: [String]
+
+    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.packageRef = packageRef
+        self.libraryName = libraryName
+        self.path = path
+        self.products = products
+        self.cModules = cModules
+    }
+}
+
 /// Helper for constructing a package following the convention system.
 ///
 /// The 'builder' here refers to the builder pattern and not any build system
@@ -295,6 +321,9 @@ public final class PackageBuilder {
     /// Information concerning the different downloaded or local (archived) binary target artifacts.
     private let binaryArtifacts: [String: BinaryArtifact]
 
+    /// Prebuilts that may referenced from this package's targets
+    private let prebuilts: [PackageIdentity: [String: PrebuiltLibrary]]
+
     /// Create multiple test products.
     ///
     /// If set to true, one test product will be created for each test target.
@@ -351,6 +380,7 @@ public final class PackageBuilder {
         path: AbsolutePath,
         additionalFileRules: [FileRuleDescription],
         binaryArtifacts: [String: BinaryArtifact],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         testEntryPointPath: AbsolutePath? = nil,
         warnAboutImplicitExecutableTargets: Bool = true,
@@ -365,6 +395,7 @@ public final class PackageBuilder {
         self.packagePath = path
         self.additionalFileRules = additionalFileRules
         self.binaryArtifacts = binaryArtifacts
+        self.prebuilts = prebuilts
         self.shouldCreateMultipleTestProducts = shouldCreateMultipleTestProducts
         self.testEntryPointPath = testEntryPointPath
         self.createREPLProduct = createREPLProduct
@@ -1211,6 +1242,33 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
+        // Add in flags for prebuilts
+        let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
+            guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
+                  let package = package,
+                  let prebuilt = prebuilts[.plain(package)]?[name]
+            else {
+                return
+            }
+
+            $0[prebuilt.libraryName] = prebuilt
+        }
+
+        for prebuilt in prebuiltLibraries.values {
+            let libDir = prebuilt.path.appending(component: "lib").pathString
+            var ldFlagsAssignment = BuildSettings.Assignment()
+            ldFlagsAssignment.values = ["\(libDir)/lib\(prebuilt.libraryName).a"]
+            table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
+
+            var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
+            for cModule in prebuilt.cModules {
+                includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+            }
+            var includeAssignment = BuildSettings.Assignment()
+            includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
+            table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
+        }
+
         return table
     }
 

Sources/Workspace/CMakeLists.txt

@@ -27,6 +27,7 @@ add_library(Workspace
   Workspace+Editing.swift
   Workspace+Manifests.swift
   Workspace+PackageContainer.swift
+  Workspace+Prebuilts.swift
   Workspace+Registry.swift
   Workspace+ResolvedPackages.swift
   Workspace+Signing.swift

Sources/Workspace/ManagedPrebuilt.swift

@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Basics
+import PackageModel
+
+extension Workspace {
+    /// A downloaded prebuilt managed by the workspace.
+    public struct ManagedPrebuilt {
+        /// The package reference.
+        public let packageRef: PackageReference
+
+        /// The name of the binary target the artifact corresponds to.
+        public let libraryName: String
+
+        /// The path to the extracted prebuilt artifacts
+        public let path: AbsolutePath
+
+        /// The products in the library
+        public let products: [String]
+
+        /// The C modules that need their includes directory added to the include path
+        public let cModules: [String]
+    }
+}
+
+extension Workspace.ManagedPrebuilt: CustomStringConvertible {
+    public var description: String {
+        return "<ManagedArtifact: \(self.packageRef.identity).\(self.libraryName)>"
+    }
+}
+
+// MARK: - ManagedArtifacts
+
+extension Workspace {
+    /// A collection of managed artifacts which have been downloaded.
+    public final class ManagedPrebuilts {
+        /// A mapping from package identity, to target name, to ManagedArtifact.
+        private var artifactMap: [PackageIdentity: [String: ManagedPrebuilt]]
+
+        internal var artifacts: AnyCollection<ManagedPrebuilt> {
+            AnyCollection(self.artifactMap.values.lazy.flatMap{ $0.values })
+        }
+
+        init() {
+            self.artifactMap = [:]
+        }
+
+        init(_ artifacts: [ManagedPrebuilt]) throws {
+            let artifactsByPackagePath = Dictionary(grouping: artifacts, by: { $0.packageRef.identity })
+            self.artifactMap = try artifactsByPackagePath.mapValues{ artifacts in
+                // rdar://86857825 do not use Dictionary(uniqueKeysWithValues:) as it can crash the process when input is incorrect such as in older versions of SwiftPM
+                var map = [String: ManagedPrebuilt]()
+                for artifact in artifacts {
+                    if map[artifact.libraryName] != nil {
+                        throw StringError("binary artifact for '\(artifact.libraryName)' already exists in managed artifacts")
+                    }
+                    map[artifact.libraryName] = artifact
+                }
+                return map
+            }
+        }
+
+        public subscript(packageIdentity packageIdentity: PackageIdentity, targetName targetName: String) -> ManagedPrebuilt? {
+            self.artifactMap[packageIdentity]?[targetName]
+        }
+
+        public func add(_ artifact: ManagedPrebuilt) {
+            self.artifactMap[artifact.packageRef.identity, default: [:]][artifact.libraryName] = artifact
+        }
+
+        public func remove(packageIdentity: PackageIdentity, targetName: String) {
+            self.artifactMap[packageIdentity]?[targetName] = nil
+        }
+    }
+}
+
+extension Workspace.ManagedPrebuilts: Collection {
+    public var startIndex: AnyIndex {
+        self.artifacts.startIndex
+    }
+
+    public var endIndex: AnyIndex {
+        self.artifacts.endIndex
+    }
+
+    public subscript(index: AnyIndex) -> Workspace.ManagedPrebuilt {
+        self.artifacts[index]
+    }
+
+    public func index(after index: AnyIndex) -> AnyIndex {
+        self.artifacts.index(after: index)
+    }
+}
+
+extension Workspace.ManagedPrebuilts: CustomStringConvertible {
+    public var description: String {
+        "<ManagedArtifacts: \(Array(self.artifacts))>"
+    }
+}