From 07df8c3db8da93756fb2386184701fb1d7b3ec02 Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Fri, 8 Nov 2024 08:14:33 +0100 Subject: [PATCH] Improve Android SDK handling (#3913) This PR seeks to improve Android SDK handling approach. The current issues are: * Android SDK setup is [limited only to Linux](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L24) and is also using [tooling which may be not available on other platforms](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L102-L109). * Each module [will have its own copy of SDK](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L114). Imagine a project with 20 Android modules (19 lib modules + 1 app module), it will be 20x space usage than it should be. This is quite significant, considering that the basic setup (`platforms:android-35`, `platform-tools` and `build-tools;35.0.0`) is taking ~300 MB. * Licenses are [accepted on the end-user behalf](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L108), without the consent. This PR: * Aligns the SDK setup experience with AGP (Android Gradle Plugin): * Expects Android SDK (at least command line tools) to be present on the end-user machine and registered with the env variable (`ANDROID_HOME`). * Expects command line tools to be present there to utilize [sdkmanager](https://developer.android.com/tools/sdkmanager) to setup the necessary SDK components. * Expects user to accept the necessary licenses by themselves. If they are not accepted, `sdkmanager` will block `stdin` waiting for the user input before proceeding with installation. **Note**: when downloading [Android Studio](https://developer.android.com/studio), it will come with the necessary basic license already accepted + some basic SDK components. If downloading command line tools, it is necessary to accept the main license (`android-sdk-license`) manually. * Installs all the necessary components to the location pointed by the `ANDROID_HOME` env variable, so that they are shared by all modules. * Adds the necessary step on the CI to install Android SDK for the jobs where it is needed. **Note**: I was also thinking about extracting Android-related examples into a dedicated job, but then for the existing jobs doing something like `example.kotlinlib[__].local.testCached` ->`example.kotlinlib[__:^AndroidAppKotlinModule].local.testCached` doesn't work (at least `mill resolve` still shows Android module), all cross-segments should be listed manually. Speaking of `sdkmanager`: it is also possible to utilize directly the [following artifact](https://mvnrepository.com/artifact/com.android.tools/sdklib) and put it in a worker, but for this one it is impossible to find a version on the Android Developers website, so end-user will have troubles updating it and anyway it is one-shot action normally, so keeping it even in the separate classpath looks like an overkill. [Command line tools](https://developer.android.com/tools) will be needed anyway further for the other tools it provides. Co-authored-by: 0xnm <0xnm@users.noreply.github.com> --- .github/workflows/run-mill-action.yml | 7 + .github/workflows/run-tests.yml | 3 + .../javalib/android/AndroidSdkModule.scala | 184 +++++++++++++----- 3 files changed, 149 insertions(+), 45 deletions(-) diff --git a/.github/workflows/run-mill-action.yml b/.github/workflows/run-mill-action.yml index 2eaee6f62a6..4e3b6c6744f 100644 --- a/.github/workflows/run-mill-action.yml +++ b/.github/workflows/run-mill-action.yml @@ -27,6 +27,9 @@ on: env-bridge-versions: default: 'none' type: string + install-android-sdk: + default: false + type: boolean jobs: run: @@ -62,6 +65,10 @@ jobs: with: node-version: '22' + - uses: android-actions/setup-android@v3 + if: ${{ inputs.install-android-sdk }} + with: + log-accepted-android-sdk-licenses: false - name: Prepare git config run: | diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b38301903d1..4ed34462d82 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -65,10 +65,12 @@ jobs: millargs: "contrib.__.testCached" - java-version: 17 + install-android-sdk: true millargs: "'example.javalib.__.local.testCached'" - java-version: 17 millargs: "'example.scalalib.__.local.testCached'" - java-version: 17 + install-android-sdk: true millargs: "'example.kotlinlib.__.local.testCached'" - java-version: '11' millargs: "'example.thirdparty[{mockito,acyclic,commons-io}].local.testCached'" @@ -93,6 +95,7 @@ jobs: uses: ./.github/workflows/run-mill-action.yml with: + install-android-sdk: ${{ matrix.install-android-sdk }} java-version: ${{ matrix.java-version }} millargs: ${{ matrix.millargs }} diff --git a/scalalib/src/mill/javalib/android/AndroidSdkModule.scala b/scalalib/src/mill/javalib/android/AndroidSdkModule.scala index 20b7ae4afa4..f110d60f13d 100644 --- a/scalalib/src/mill/javalib/android/AndroidSdkModule.scala +++ b/scalalib/src/mill/javalib/android/AndroidSdkModule.scala @@ -2,6 +2,11 @@ package mill.javalib.android import mill._ +import java.math.BigInteger +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import scala.xml.XML + /** * Trait for managing the Android SDK in a Mill build system. * @@ -17,12 +22,9 @@ import mill._ @mill.api.experimental trait AndroidSdkModule extends Module { - /** - * Provides the URL to download the Android SDK command-line tools. - */ - def sdkUrl: T[String] = Task { - "https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip" - } + // this has a format `repository2-%d`, where the last number is schema version. For the needs of this module it + // is okay to stick with the particular version. + private val remotePackagesUrl = "https://dl.google.com/android/repository/repository2-3.xml" /** * Specifies the version of the Android build tools to be used. @@ -38,14 +40,17 @@ trait AndroidSdkModule extends Module { * Provides the path to the `android.jar` file, necessary for compiling Android apps. */ def androidJarPath: T[PathRef] = Task { - PathRef(installAndroidSdk().path / "platforms" / platformsVersion().toString / "android.jar") + installAndroidSdkComponents() + PathRef(sdkPath().path / "platforms" / platformsVersion() / "android.jar") } /** * Provides path to the Android build tools for the selected version. */ - def buildToolsPath: T[PathRef] = - Task { PathRef(installAndroidSdk().path / "build-tools" / buildToolsVersion().toString) } + def buildToolsPath: T[PathRef] = Task { + installAndroidSdkComponents() + PathRef(sdkPath().path / "build-tools" / buildToolsVersion()) + } /** * Provides path to D8 Dex compiler, used for converting Java bytecode into Dalvik bytecode. @@ -76,46 +81,135 @@ trait AndroidSdkModule extends Module { } /** - * Installs the Android SDK by performing the following actions: + * Installs the necessary Android SDK components such as platform-tools, build-tools, and Android platforms. * - * - Downloads the SDK command-line tools from the specified URL. - * - * - Extracts the downloaded zip file into the SDK directory. - * - * - Accepts the required SDK licenses. - * - * - Installs essential SDK components such as platform-tools, build-tools, and Android platforms. - * - * For more details on the sdkmanager tool, refer to: + * For more details on the `sdkmanager` tool, refer to: * [[https://developer.android.com/tools/sdkmanager sdkmanager Documentation]] * - * @return A task containing a `PathRef` pointing to the installed SDK directory. + * @return A task containing a [[PathRef]] pointing to the SDK directory. */ - def installAndroidSdk: T[PathRef] = Task { - val zipFilePath: os.Path = T.dest / "commandlinetools.zip" - val sdkManagerPath: os.Path = T.dest / "cmdline-tools/bin/sdkmanager" - - // Download SDK command-line tools - os.write(zipFilePath, requests.get(sdkUrl().toString).bytes) - - // Extract the downloaded SDK tools into the destination directory - os.call(Seq("unzip", zipFilePath.toString, "-d", T.dest.toString)) - - // Automatically accept the SDK licenses - os.call(Seq( - "bash", - "-c", - s"yes | $sdkManagerPath --licenses --sdk_root=${T.dest}" - )) - - // Install platform-tools, build-tools, and the Android platform - os.call(Seq( - sdkManagerPath.toString, - s"--sdk_root=${T.dest}", + def installAndroidSdkComponents: T[Unit] = Task { + val sdkPath0 = sdkPath() + val sdkManagerPath = findLatestSdkManager(sdkPath0.path) match { + case Some(x) => x + case _ => throw new IllegalStateException( + s"Cannot locate cmdline-tools in Android SDK $sdkPath0. Download" + + " it at https://developer.android.com/studio#command-tools. See https://developer.android.com/tools" + + " for more details." + ) + } + + val packages = Seq( "platform-tools", - s"build-tools;${buildToolsVersion().toString}", - s"platforms;${platformsVersion().toString}" - )) - PathRef(T.dest) + s"build-tools;${buildToolsVersion()}", + s"platforms;${platformsVersion()}" + ) + // sdkmanager executable and state of the installed package is a shared resource, which can be accessed + // from the different Android SDK modules. + AndroidSdkLock.synchronized { + val missingPackages = packages.filter(p => !isPackageInstalled(sdkPath0.path, p)) + val packagesWithoutLicense = missingPackages + .map(p => (p, isLicenseAccepted(sdkPath0.path, remoteReposInfo().path, p))) + .filter(!_._2) + if (packagesWithoutLicense.nonEmpty) { + throw new IllegalStateException( + "Failed to install the following SDK packages, because their respective" + + s" licenses are not accepted:\n\n${packagesWithoutLicense.map(_._1).mkString("\n")}" + ) + } + + if (missingPackages.nonEmpty) { + val callResult = os.call( + // Install platform-tools, build-tools, and the Android platform + Seq(sdkManagerPath.toString) ++ missingPackages, + stdout = os.Inherit + ) + if (callResult.exitCode != 0) { + throw new IllegalStateException( + "Failed to install Android SDK components. Check logs for more details." + ) + } + } + } + } + + private def sdkPath: T[PathRef] = Task { + T.env.get("ANDROID_HOME") + .orElse(T.env.get("ANDROID_SDK_ROOT")) match { + case Some(x) => PathRef(os.Path(x)) + case _ => throw new IllegalStateException("Android SDK location not found. Define a valid" + + " SDK location with an ANDROID_HOME environment variable.") + } + } + + private def isPackageInstalled(sdkPath: os.Path, packageName: String): Boolean = + os.exists(sdkPath / os.SubPath(packageName.replaceAll(";", "/"))) + + private def isLicenseAccepted( + sdkPath: os.Path, + remoteReposInfo: os.Path, + packageName: String + ): Boolean = { + val (licenseName, licenseHash) = licenseForPackage(remoteReposInfo, packageName) + val licenseFile = sdkPath / "licenses" / licenseName + os.exists(licenseFile) && os.isFile(licenseFile) && os.read(licenseFile).contains(licenseHash) + } + + private def licenseForPackage(remoteReposInfo: os.Path, packageName: String): (String, String) = { + val repositoryInfo = XML.loadFile(remoteReposInfo.toIO) + val remotePackage = (repositoryInfo \ "remotePackage") + .filter(_ \@ "path" == packageName) + .head + val licenseName = (remotePackage \ "uses-license").head \@ "ref" + val licenseText = (repositoryInfo \ "license") + .filter(_ \@ "id" == licenseName) + .text + .replaceAll( + "(?<=\\s)[ \t]*", + "" + ) // remove spaces and tabs preceded by space, tab, or newline. + .replaceAll("(? + val candidates = os.list(sdkPath / "cmdline-tools") + .filter(os.isDir) + if (candidates.nonEmpty) { + val latestCmdlineToolsPath = candidates + .map(p => (p, p.baseName.split('.'))) + .filter(_._2 match { + case Array(_, _) => true + case _ => false + }) + .maxBy(_._2.head.toInt)._1 + sdkManagerPath = latestCmdlineToolsPath / "bin" / "sdkmanager" + } + } + Some(sdkManagerPath).filter(os.exists) } } + +private object AndroidSdkLock