Add support for a prebuilt swift-syntax library for macros

Package.swift

@@ -655,6 +655,15 @@ let package = Package(
             name: "swift-package-registry",
             dependencies: ["Commands", "PackageRegistryCommand"]
         ),
+        .executableTarget(
+            /** Utility to produce the artifacts for prebuilts */
+            name: "swift-build-prebuilts",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "Basics",
+                "Workspace",
+            ]
+        ),
 
         // MARK: Support for Swift macros, should eventually move to a plugin-based solution
 

Sources/Basics/Archiver/ZipArchiver.swift

@@ -13,6 +13,10 @@
 import Dispatch
 import struct TSCBasic.FileSystemError
 
+#if os(Windows)
+import WinSDK
+#endif
+
 /// An `Archiver` that handles ZIP archives using the command-line `zip` and `unzip` tools.
 public struct ZipArchiver: Archiver, Cancellable {
     public var supportedExtensions: Set<String> { ["zip"] }
@@ -23,6 +27,11 @@ public struct ZipArchiver: Archiver, Cancellable {
     /// Helper for cancelling in-flight requests
     private let cancellator: Cancellator
 
+    /// Absolute path to the Windows tar in the system folder
+    #if os(Windows)
+    private let windowsTar: String
+    #endif
+
     /// Creates a `ZipArchiver`.
     ///
     /// - Parameters:
@@ -31,6 +40,19 @@ public struct ZipArchiver: Archiver, Cancellable {
     public init(fileSystem: FileSystem, cancellator: Cancellator? = .none) {
         self.fileSystem = fileSystem
         self.cancellator = cancellator ?? Cancellator(observabilityScope: .none)
+
+        #if os(Windows)
+        var tarPath: PWSTR?
+        defer { CoTaskMemFree(tarPath) }
+        let hr = withUnsafePointer(to: FOLDERID_System) { id in
+            SHGetKnownFolderPath(id, DWORD(KF_FLAG_DEFAULT.rawValue), nil, &tarPath)
+        }
+        if hr == S_OK, let tarPath {
+            windowsTar = String(decodingCString: tarPath, as: UTF16.self) + "\\tar.exe"
+        } else {
+            windowsTar = "tar.exe"
+        }
+        #endif
     }
 
     public func extract(
@@ -48,7 +70,9 @@ public struct ZipArchiver: Archiver, Cancellable {
             }
 
             #if os(Windows)
-            let process = AsyncProcess(arguments: ["tar.exe", "xf", archivePath.pathString, "-C", destinationPath.pathString])
+            // FileManager lost the ability to detect tar.exe as executable.
+            // It's part of system32 anyway so use the absolute path.
+            let process = AsyncProcess(arguments: [windowsTar, "xf", archivePath.pathString, "-C", destinationPath.pathString])
             #else
             let process = AsyncProcess(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString])
             #endif
@@ -82,7 +106,7 @@ public struct ZipArchiver: Archiver, Cancellable {
         #if os(Windows)
         let process = AsyncProcess(
             // FIXME: are these the right arguments?
-            arguments: ["tar.exe", "-a", "-c", "-f", destinationPath.pathString, directory.basename],
+            arguments: [windowsTar, "-a", "-c", "-f", destinationPath.pathString, directory.basename],
             workingDirectory: directory.parentDirectory
         )
         #else
@@ -121,7 +145,7 @@ public struct ZipArchiver: Archiver, Cancellable {
             }
 
             #if os(Windows)
-            let process = AsyncProcess(arguments: ["tar.exe", "tf", path.pathString])
+            let process = AsyncProcess(arguments: [windowsTar, "tf", path.pathString])
             #else
             let process = AsyncProcess(arguments: ["unzip", "-t", path.pathString])
             #endif

Sources/Basics/Collections/IdentifiableSet.swift

@@ -122,3 +122,9 @@ extension IdentifiableSet: Hashable {
         }
     }
 }
+
+extension IdentifiableSet: ExpressibleByArrayLiteral {
+    public init(arrayLiteral elements: Element...) {
+        self.init(elements)
+    }
+}

Sources/Commands/CommandWorkspaceDelegate.swift

@@ -182,6 +182,42 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate {
         self.progressHandler(step, total, "Downloading \(artifacts)")
     }
 
+    /// The workspace has started downloading a binary artifact.
+    func willDownloadPrebuilt(from url: String, fromCache: Bool) {
+        if fromCache {
+            self.outputHandler("Fetching package prebuilt \(url) from cache", false)
+        } else {
+            self.outputHandler("Downloading package prebuilt \(url)", false)
+        }
+    }
+
+    /// The workspace has finished downloading a binary artifact.
+    func didDownloadPrebuilt(
+        from url: String,
+        result: Result<(path: AbsolutePath, fromCache: Bool), Error>,
+        duration: DispatchTimeInterval
+    ) {
+        guard case .success(let fetchDetails) = result, !self.observabilityScope.errorsReported else {
+            return
+        }
+
+        if fetchDetails.fromCache {
+            self.outputHandler("Fetched \(url) from cache (\(duration.descriptionInSeconds))", false)
+        } else {
+            self.outputHandler("Downloaded \(url) (\(duration.descriptionInSeconds))", false)
+        }
+    }
+
+    /// The workspace is downloading a binary artifact.
+    func downloadingPrebuilt(from url: String, bytesDownloaded: Int64, totalBytesToDownload: Int64?) {
+
+    }
+
+    /// The workspace finished downloading all binary artifacts.
+    func didDownloadAllPrebuilts() {
+
+    }
+
     // registry signature handlers
 
     func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version, completion: (Bool) -> Void) {

Sources/CoreCommands/Options.swift

@@ -186,6 +186,12 @@ public struct CachingOptions: ParsableArguments {
             self.init(rawValue: argument)
         }
     }
+
+    /// Whether to use macro prebuilts or not
+    @Flag(name: .customLong("experimental-prebuilts"),
+          inversion: .prefixedEnableDisable,
+          help: "Whether to use prebuilt swift-syntax libraries for macros")
+    public var usePrebuilts: Bool = false
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -80,6 +80,7 @@ public typealias WorkspaceDelegateProvider = (
     _ progressHandler: @escaping (Int64, Int64, String?) -> Void,
     _ inputHandler: @escaping (String, (String?) -> Void) -> Void
 ) -> WorkspaceDelegate
+
 public typealias WorkspaceLoaderProvider = (_ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope)
     -> WorkspaceLoader
 
@@ -470,7 +471,8 @@ public final class SwiftCommandState {
                     // TODO: should supportsAvailability be a flag as well?
                     .init(url: $0, supportsAvailability: true)
                 },
-                manifestImportRestrictions: .none
+                manifestImportRestrictions: .none,
+                usePrebuilts: options.caching.usePrebuilts
             ),
             cancellator: self.cancellator,
             initializationWarningHandler: { self.observabilityScope.emit(warning: $0) },

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -29,6 +29,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         customPlatformsRegistry: PlatformRegistry? = .none,
@@ -47,6 +48,7 @@ extension ModulesGraph {
             requiredDependencies: requiredDependencies,
             unsafeAllowedPackages: unsafeAllowedPackages,
             binaryArtifacts: binaryArtifacts,
+            prebuilts: prebuilts,
             shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
             createREPLProduct: createREPLProduct,
             traitConfiguration: nil,
@@ -69,6 +71,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]], // Product name to library mapping
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         traitConfiguration: TraitConfiguration? = nil,
@@ -212,7 +215,8 @@ extension ModulesGraph {
                     productFilter: node.productFilter,
                     path: packagePath,
                     additionalFileRules: additionalFileRules,
-                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],                
+                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],
+                    prebuilts: prebuilts,
                     shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
                     testEntryPointPath: testEntryPointPath,
                     createREPLProduct: manifest.packageKind.isRoot ? createREPLProduct : false,
@@ -246,6 +250,7 @@ extension ModulesGraph {
             manifestToPackage: manifestToPackage,
             rootManifests: root.manifests,
             unsafeAllowedPackages: unsafeAllowedPackages,
+            prebuilts: prebuilts,
             platformRegistry: customPlatformsRegistry ?? .default,
             platformVersionProvider: platformVersionProvider,
             fileSystem: fileSystem,
@@ -377,6 +382,7 @@ private func createResolvedPackages(
     // FIXME: This shouldn't be needed once <rdar://problem/33693433> is fixed.
     rootManifests: [PackageIdentity: Manifest],
     unsafeAllowedPackages: Set<PackageReference>,
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
     platformRegistry: PlatformRegistry,
     platformVersionProvider: PlatformVersionProvider,
     fileSystem: FileSystem,
@@ -665,6 +671,11 @@ private func createResolvedPackages(
                     }
                 }
 
+                if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
+                    // using a prebuilt instead.
+                    continue
+                }
+
                 // Find the product in this package's dependency products.
                 // Look it up by ID if module aliasing is used, otherwise by name.
                 let product = lookupByProductIDs ? productDependencyMap[productRef.identity] : productDependencyMap[productRef.name]

Sources/PackageGraph/ModulesGraph.swift

@@ -419,6 +419,7 @@ public func loadModulesGraph(
     fileSystem: FileSystem,
     manifests: [Manifest],
     binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]] = [:],
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]] = [:],
     explicitProduct: String? = .none,
     shouldCreateMultipleTestProducts: Bool = false,
     createREPLProduct: Bool = false,
@@ -451,6 +452,7 @@ public func loadModulesGraph(
             .swiftpmFileTypes,
         externalManifests: externalManifests,
         binaryArtifacts: binaryArtifacts,
+        prebuilts: prebuilts,
         shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
         createREPLProduct: createREPLProduct,
         customXCTestMinimumDeploymentTargets: customXCTestMinimumDeploymentTargets,

Sources/PackageLoading/PackageBuilder.swift

@@ -268,6 +268,32 @@ public struct BinaryArtifact {
     }
 }
 
+/// A structure representing a prebuilt library to be used instead of a source dependency
+public struct PrebuiltLibrary {
+    /// The package reference.
+    public let packageRef: PackageReference
+
+    /// The name of the binary target the artifact corresponds to.
+    public let libraryName: String
+
+    /// The path to the extracted prebuilt artifacts
+    public let path: AbsolutePath
+
+    /// The products in the library
+    public let products: [String]
+
+    /// The C modules that need their includes directory added to the include path
+    public let cModules: [String]
+
+    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.packageRef = packageRef
+        self.libraryName = libraryName
+        self.path = path
+        self.products = products
+        self.cModules = cModules
+    }
+}
+
 /// Helper for constructing a package following the convention system.
 ///
 /// The 'builder' here refers to the builder pattern and not any build system
@@ -295,6 +321,9 @@ public final class PackageBuilder {
     /// Information concerning the different downloaded or local (archived) binary target artifacts.
     private let binaryArtifacts: [String: BinaryArtifact]
 
+    /// Prebuilts that may referenced from this package's targets
+    private let prebuilts: [PackageIdentity: [Product.ID: PrebuiltLibrary]]
+
     /// Create multiple test products.
     ///
     /// If set to true, one test product will be created for each test target.
@@ -351,6 +380,7 @@ public final class PackageBuilder {
         path: AbsolutePath,
         additionalFileRules: [FileRuleDescription],
         binaryArtifacts: [String: BinaryArtifact],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         testEntryPointPath: AbsolutePath? = nil,
         warnAboutImplicitExecutableTargets: Bool = true,
@@ -365,6 +395,7 @@ public final class PackageBuilder {
         self.packagePath = path
         self.additionalFileRules = additionalFileRules
         self.binaryArtifacts = binaryArtifacts
+        self.prebuilts = prebuilts
         self.shouldCreateMultipleTestProducts = shouldCreateMultipleTestProducts
         self.testEntryPointPath = testEntryPointPath
         self.createREPLProduct = createREPLProduct
@@ -1211,6 +1242,33 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
+        // Add in flags for prebuilts
+        let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
+            guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
+                  let package = package,
+                  let prebuilt = prebuilts[.plain(package)]?[name]
+            else {
+                return
+            }
+
+            $0[prebuilt.libraryName] = prebuilt
+        }
+
+        for prebuilt in prebuiltLibraries.values {
+            let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
+            var ldFlagsAssignment = BuildSettings.Assignment()
+            ldFlagsAssignment.values = [lib]
+            table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
+
+            var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
+            for cModule in prebuilt.cModules {
+                includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+            }
+            var includeAssignment = BuildSettings.Assignment()
+            includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
+            table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
+        }
+
         return table
     }
 

Sources/PackageModel/Product.swift

@@ -12,10 +12,12 @@
 
 import Basics
 
-public class Product {
+public class Product: Identifiable {
     /// The name of the product.
     public let name: String
 
+    public var id: String { name }
+
     /// Fully qualified name for this product: package ID + name of this product
     public let identity: String
 

Sources/Workspace/CMakeLists.txt

@@ -13,6 +13,7 @@ add_library(Workspace
   LoadableResult.swift
   ManagedArtifact.swift
   ManagedDependency.swift
+  ManagedPrebuilt.swift
   PackageContainer/FileSystemPackageContainer.swift
   PackageContainer/RegistryPackageContainer.swift
   PackageContainer/SourceControlPackageContainer.swift
@@ -27,6 +28,7 @@ add_library(Workspace
   Workspace+Editing.swift
   Workspace+Manifests.swift
   Workspace+PackageContainer.swift
+  Workspace+Prebuilts.swift
   Workspace+Registry.swift
   Workspace+ResolvedPackages.swift
   Workspace+Signing.swift