Merge branch 'main' into swiftlang#65-support-armv7

README.md

@@ -29,9 +29,8 @@ installed, in case you've used the `swift experimental-sdk install` command befo
 
 ## Supported platforms and minimum versions
 
-macOS as a host platform and a few Linux distributions as target platforms are supported by the generator.
-Support for Linux as a host platform is currently in development. Eventually, the generator will allow cross-compiling between any
-Linux distributions officially supported by the Swift project.
+macOS as a host platform and Linux as both host and target platforms are supported by the generator.
+The generator also allows cross-compiling between any Linux distributions officially supported by the Swift project.
 
 | Platform       | Supported Version as Host | Supported Version as Target |
 | -:             | :-                        | :-                          |
@@ -125,7 +124,7 @@ swift run swift-sdk-generator make-linux-sdk --with-docker --linux-distribution-
 You can also specify the base container image by name:
 
 ```
-swift run swift-sdk-generator make-linux-sdk --with-docker --from-container-image swift:5.9-jammy
+swift run swift-sdk-generator make-linux-sdk --from-container-image swift:5.9-jammy
 ```
 
 ```

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -17,19 +17,29 @@ import struct SystemPackage.FilePath
 
 @main
 struct GeneratorCLI: AsyncParsableCommand {
+  static let appLogger = Logger(label: "org.swift.swift-sdk-generator")
+
   static let configuration = CommandConfiguration(
     commandName: "swift-sdk-generator",
     subcommands: [MakeLinuxSDK.self, MakeWasmSDK.self],
     defaultSubcommand: MakeLinuxSDK.self
   )
 
+  static func loggerWithLevel(from options: GeneratorOptions) -> Logger {
+    var logger = self.appLogger
+    if options.verbose {
+      logger.logLevel = .debug
+    }
+    return logger
+  }
+
   static func run(
     recipe: some SwiftSDKRecipe,
     targetTriple: Triple,
     options: GeneratorOptions
   ) async throws {
+    let logger = loggerWithLevel(from: options)
     let elapsed = try await ContinuousClock().measure {
-      let logger = Logger(label: "org.swift.swift-sdk-generator")
       let generator = try await SwiftSDKGenerator(
         bundleVersion: options.bundleVersion,
         targetTriple: targetTriple,
@@ -57,7 +67,7 @@ struct GeneratorCLI: AsyncParsableCommand {
       try await generatorTask.value
     }
 
-    print("\nTime taken for this generator run: \(elapsed.intervalString).")
+    logger.info("Generator run finished successfully.", metadata: ["elapsedTime": .string(elapsed.intervalString)])
   }
 }
 
@@ -115,7 +125,7 @@ extension GeneratorCLI {
       but requires exactly the same version of the swift.org toolchain to be installed for it to work.
       """
     )
-    var hostToolchain: Bool = true
+    var hostToolchain: Bool = false
 
     @Option(
       help: """
@@ -151,7 +161,7 @@ extension GeneratorCLI {
       let current = try SwiftSDKGenerator.getCurrentTriple(isVerbose: self.verbose)
       if let arch = hostArch {
         let target = Triple(arch: arch, vendor: current.vendor!, os: current.os!)
-        print("deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`")
+        appLogger.warning("deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`")
         return target
       }
       return current
@@ -202,14 +212,14 @@ extension GeneratorCLI {
       }
       if let arch = generatorOptions.targetArch {
         let target = Triple(arch: arch, vendor: nil, os: .linux, environment: .gnu)
-        print("deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`")
+        appLogger.warning("deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`")
       }
       return Triple(arch: hostTriple.arch!, vendor: nil, os: .linux, environment: .gnu)
     }
 
     func run() async throws {
       if self.isInvokedAsDefaultSubcommand() {
-        print(
+        appLogger.warning(
           "deprecated: Please explicitly specify the subcommand to run. For example: $ swift-sdk-generator make-linux-sdk"
         )
       }
@@ -236,7 +246,8 @@ extension GeneratorCLI {
         fromContainerImage: self.fromContainerImage,
         hostSwiftPackagePath: self.generatorOptions.hostSwiftPackagePath,
         targetSwiftPackagePath: self.generatorOptions.targetSwiftPackagePath,
-        includeHostToolchain: self.generatorOptions.hostToolchain
+        includeHostToolchain: self.generatorOptions.hostToolchain,
+        logger: loggerWithLevel(from: self.generatorOptions)
       )
       try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions)
     }
@@ -290,7 +301,8 @@ extension GeneratorCLI {
         },
         targetSwiftPackagePath: FilePath(targetSwiftPackagePath),
         wasiSysroot: FilePath(self.wasiSysroot),
-        swiftVersion: self.generatorOptions.swiftVersion
+        swiftVersion: self.generatorOptions.swiftVersion,
+        logger: loggerWithLevel(from: self.generatorOptions)
       )
       let targetTriple = self.deriveTargetTriple()
       try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions)

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift

@@ -18,7 +18,7 @@ extension SwiftSDKGenerator {
     baseDockerImage: String,
     sdkDirPath: FilePath
   ) async throws {
-    logGenerationStep("Launching a Docker container to copy Swift SDK for the target triple from it...")
+    logger.info("Launching a container to extract the Swift SDK for the target triple...")
     try await withDockerContainer(fromImage: baseDockerImage) { containerID in
       try await inTemporaryDirectory { generator, _ in
         let sdkUsrPath = sdkDirPath.appending("usr")
@@ -107,7 +107,7 @@ extension SwiftSDKGenerator {
   }
 
   func copyTargetSwift(from distributionPath: FilePath, sdkDirPath: FilePath) async throws {
-    logGenerationStep("Copying Swift core libraries for the target triple into Swift SDK bundle...")
+    logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...")
 
     for (pathWithinPackage, pathWithinSwiftSDK, ignoreIfMissing) in [
       ("lib/swift", sdkDirPath.appending("usr/lib"), false),

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import AsyncAlgorithms
+import Logging
 import Helpers
 import RegexBuilder
 
@@ -23,13 +24,19 @@ import struct SystemPackage.FilePath
 private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu"
 private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports"
 
+extension FilePath {
+  var metadataValue: Logger.MetadataValue {
+    .string(self.string)
+  }
+}
+
 extension SwiftSDKGenerator {
   func downloadArtifacts(
     _ client: some HTTPClientProtocol, _ engine: QueryEngine,
     downloadableArtifacts: inout DownloadableArtifacts,
     itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item]
   ) async throws {
-    logGenerationStep("Downloading required toolchain packages...")
+    logger.info("Downloading required toolchain packages...")
     let hostLLVMURL = downloadableArtifacts.hostLLVM.remoteURL
     // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC.
     let isLLVMBinaryArtifactAvailable = try await type(of: client).with(http1Only: true) {
@@ -46,7 +53,7 @@ extension SwiftSDKGenerator {
     let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
       for item in itemsToDownload(downloadableArtifacts) {
         group.addTask {
-          try await engine[DownloadArtifactQuery(artifact: item, httpClient: client)]
+          try await engine[DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger)]
         }
       }
 
@@ -57,10 +64,10 @@ extension SwiftSDKGenerator {
       return result
     }
 
-    print("Using downloaded artifacts in these locations:")
-    for path in results.map(\.path) {
-      print(path)
-    }
+    logger.info("Using downloaded artifacts from cache")
+    logger.debug("Using downloaded artifacts in these locations.", metadata: [
+      "paths": .array(results.map(\.path.metadataValue))
+    ])
   }
 
   func downloadUbuntuPackages(
@@ -70,7 +77,7 @@ extension SwiftSDKGenerator {
     versionsConfiguration: VersionsConfiguration,
     sdkDirPath: FilePath
   ) async throws {
-    logGenerationStep("Parsing Ubuntu packages list...")
+    logger.debug("Parsing Ubuntu packages list...")
 
     async let mainPackages = try await client.parseUbuntuPackagesList(
       ubuntuRelease: versionsConfiguration.linuxDistribution.release,
@@ -108,12 +115,13 @@ extension SwiftSDKGenerator {
       )
     }
 
-    print("Downloading \(urls.count) Ubuntu packages...")
+    logger.info("Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)])
     try await inTemporaryDirectory { fs, tmpDir in
       let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine)
-      report(downloadedFiles: downloadedFiles)
+      await report(downloadedFiles: downloadedFiles)
 
       for fileName in urls.map(\.lastPathComponent) {
+        logger.debug("Extracting deb package...", metadata: ["fileName": .string(fileName)])
         try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath)
       }
     }
@@ -148,13 +156,16 @@ extension SwiftSDKGenerator {
       return result
     }
   }
-}
 
-private func report(downloadedFiles: [(URL, UInt64)]) {
-  let byteCountFormatter = ByteCountFormatter()
+  private func report(downloadedFiles: [(URL, UInt64)]) {
+    let byteCountFormatter = ByteCountFormatter()
 
-  for (url, bytes) in downloadedFiles {
-    print("\(url) – \(byteCountFormatter.string(fromByteCount: Int64(bytes)))")
+    for (url, bytes) in downloadedFiles {
+      logger.debug("Downloaded package", metadata: [
+        "url": .string(url.absoluteString),
+        "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes)))
+      ])
+    }
   }
 }
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift

@@ -60,20 +60,20 @@ public extension SwiftSDKGenerator {
 
         try await generateArtifactBundleManifest(hostTriples: swiftSDKProduct.hostTriples)
 
-        logGenerationStep(
+        // Extra spaces added for readability for the user
+        logger.info(
           """
+
+
           All done! Install the newly generated SDK with this command:
           swift experimental-sdk install \(pathsConfiguration.artifactBundlePath)
 
           After that, use the newly installed SDK when building with this command:
           swift build --experimental-swift-sdk \(artifactID)
+
           """
         )
       }
     }
   }
 }
-
-func logGenerationStep(_ message: String) {
-  print("\n\(message)")
-}

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift

@@ -17,7 +17,7 @@ import struct Foundation.Data
 
 extension SwiftSDKGenerator {
   func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws {
-    logGenerationStep("Fixing up absolute symlinks...")
+    logger.info("Fixing up absolute symlinks...")
 
     for (source, absoluteDestination) in try findSymlinks(at: sdkDirPath).filter({
       $1.string.hasPrefix("/")
@@ -51,7 +51,7 @@ extension SwiftSDKGenerator {
   func symlinkClangHeaders() throws {
     let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang")
     if !doesFileExist(at: swiftStaticClangPath) {
-      logGenerationStep("Symlinking clang headers...")
+      logger.info("Symlinking clang headers...")
       try self.createSymlink(
         at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"),
         pointingTo: "../swift/clang"

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift

@@ -22,7 +22,7 @@ private let encoder: JSONEncoder = {
 
 extension SwiftSDKGenerator {
   func generateToolsetJSON(recipe: SwiftSDKRecipe) throws -> FilePath {
-    logGenerationStep("Generating toolset JSON file...")
+    logger.info("Generating toolset JSON file...")
 
     let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending("toolset.json")
 
@@ -44,7 +44,7 @@ extension SwiftSDKGenerator {
   }
 
   func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) throws {
-    logGenerationStep("Generating destination JSON file...")
+    logger.info("Generating destination JSON file...")
 
     let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json")
 
@@ -87,7 +87,7 @@ extension SwiftSDKGenerator {
   }
 
   func generateArtifactBundleManifest(hostTriples: [Triple]?) throws {
-    logGenerationStep("Generating .artifactbundle manifest file...")
+    logger.info("Generating .artifactbundle info JSON file...")
 
     let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json")
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift

@@ -45,7 +45,7 @@ let unusedHostLibraries = [
 
 extension SwiftSDKGenerator {
   func unpackHostSwift(hostSwiftPackagePath: FilePath) async throws {
-    logGenerationStep("Unpacking and copying Swift binaries for the host triple...")
+    logger.info("Unpacking and copying Swift binaries for the host triple...")
     let pathsConfiguration = self.pathsConfiguration
 
     try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath)
@@ -59,14 +59,14 @@ extension SwiftSDKGenerator {
     if hostSwiftPackagePath.string.contains("tar.gz") {
       try await Shell.run(
         #"""
-        tar -xzf \#(hostSwiftPackagePath) -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --strip-components=1
+        tar -xzf "\#(hostSwiftPackagePath)" -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --strip-components=1
         """#,
         shouldLogCommands: isVerbose
       )
     } else {
       try await Shell.run(
         #"""
-        tar -x --to-stdout -f \#(hostSwiftPackagePath) \*.pkg/Payload |
+        tar -x --to-stdout -f "\#(hostSwiftPackagePath)" \*.pkg/Payload |
         tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr
         """#,
         shouldLogCommands: isVerbose
@@ -98,7 +98,7 @@ extension SwiftSDKGenerator {
     relativePathToRoot: [FilePath.Component],
     sdkDirPath: FilePath
   ) async throws {
-    logGenerationStep("Unpacking Swift distribution for the target triple...")
+    logger.info("Unpacking Swift distribution for the target triple...")
 
     try await inTemporaryDirectory { fs, tmpDir in
       try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir)
@@ -109,7 +109,7 @@ extension SwiftSDKGenerator {
   }
 
   func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) async throws {
-    logGenerationStep("Unpacking and copying `lld` linker...")
+    logger.info("Unpacking and copying `lld` linker...")
     let pathsConfiguration = self.pathsConfiguration
     let targetOS = self.targetTriple.os
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift

@@ -34,8 +34,6 @@ public actor SwiftSDKGenerator {
     isVerbose: Bool,
     logger: Logger
   ) async throws {
-    logGenerationStep("Looking up configuration values...")
-
     let sourceRoot = FilePath(#filePath)
       .removingLastComponent()
       .removingLastComponent()
@@ -60,7 +58,7 @@ public actor SwiftSDKGenerator {
   }
 
   private let fileManager = FileManager.default
-  private static let dockerCommand = "docker"
+  private static let dockerCommand = ProcessInfo.processInfo.environment["SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME"] ?? "docker"
 
   public static func getCurrentTriple(isVerbose: Bool) throws -> Triple {
     let current = UnixName.current!
@@ -118,7 +116,7 @@ public actor SwiftSDKGenerator {
       else { return }
     }
     try await Shell.run(
-      "\(Self.dockerCommand) cp \(id):\(containerPath) - | tar x -C \(localPath.removingLastComponent())",
+      #"\#(Self.dockerCommand) cp \#(id):\#(containerPath) - | tar x -C "\#(localPath.removingLastComponent())""#,
       shouldLogCommands: self.isVerbose
     )
   }
@@ -254,7 +252,11 @@ public actor SwiftSDKGenerator {
     let isVerbose = self.isVerbose
     try await self.inTemporaryDirectory { _, tmp in
       try await Shell.run(#"cd "\#(tmp)" && ar -x "\#(debFile)""#, shouldLogCommands: isVerbose)
-      try await print(Shell.readStdout("ls \(tmp)"))
+      if isVerbose {
+        let cmd = "ls \(tmp)"
+        let lsOutput = try await Shell.readStdout(cmd)
+        logger.debug("Files unpacked from deb file", metadata: ["cmd": .string(cmd), "output": .string(lsOutput)])
+      }
 
       try await Shell.run(
         #"tar -C "\#(directoryPath)" -xf "\#(tmp)"/data.tar.*"#,