Allow to generate configs for historical release of Scala

.github/actions/build-project/action.yaml

@@ -43,6 +43,10 @@ runs:
         ConfigFile=".github/workflows/buildConfig.json"
         DefaultJDK=11
         javaVersion=$(jq -r ".\"${{ inputs.project-name }}\".config.java.version // ${DefaultJDK}" $ConfigFile)
+        if [[ $javaVersion -ge 21 && ! "$scalaVersion" = 3.3.* ]]; then
+          echo "Force Java 17, Java 21 is only supported since 3.3.x"
+          javaVersion=17
+        fi
         echo "java-version=$javaVersion" >> $GITHUB_ENV
         echo "JavaVersion set to $javaVersion"
 
@@ -119,6 +123,7 @@ runs:
           touch build-logs.txt  build-summary.txt
           # Assume failure unless overwritten by a successful build
           echo 'failure' > build-status.txt 
+          echo 'unknown' > build-tool.txt
 
           /build/build-revision.sh \
             "$(config .project)" \
@@ -137,6 +142,7 @@ runs:
           mv build-logs.txt /opencb/
           mv build-status.txt /opencb/
           mv build-summary.txt /opencb/
+          mv build-tool.txt /opencb/
 
     - name: Check status
       id: check-status
@@ -211,7 +217,8 @@ runs:
             "$(config .version)" \
             "${{ inputs.scala-version }}" \
             "${{ github.run_id }}" \
-            "${{ steps.job-info.outputs.build-url }}"
+            "${{ steps.job-info.outputs.build-url }}" \
+            "$(cat build-tool.txt)"
           if [ $? != 0 ]; then 
             echo "::warning title=Indexing failure::Indexing results of ${{ inputs.project-name }} failed" 
           fi

.github/workflows/ci.yaml

@@ -37,15 +37,7 @@ jobs:
 
       - uses: coursier/setup-action@v1
         with:
-          apps: scala sbt mill:0.10.10
-
-      - name: Install scala-cli
-        run: |
-          curl -s --compressed "https://virtuslab.github.io/scala-cli-packages/KEY.gpg" | sudo apt-key add -
-          sudo curl -s --compressed -o /etc/apt/sources.list.d/scala_cli_packages.list "https://virtuslab.github.io/scala-cli-packages/debian/scala_cli_packages.list"
-          sudo apt-get update && \
-            sudo apt-get install scala-cli && \
-            scala-cli --version
+          apps: scala sbt mill:0.10.10 scala-cli:1.0.4
 
       - name: Start minikube
         run: minikube start

coordinator/.scalafmt.conf

@@ -0,0 +1,3 @@
+version = "3.7.14"
+runner.dialect = scala3
+maxColumn=100

coordinator/configs/projects-config.conf

@@ -419,7 +419,8 @@ laserdisc-io_fs2-aws{
   tests = compile-only
   sbt.commands = ["disableFatalWarnings"]
 }
-lichess-org_lila.java.version = 17
+lichess-org_lila.java.version = 21
+lichess-org_playframework-lila.java.version = 21
 lichess-org_scalachess.tests = compile-only // Deadlocks in containers
 linyxus_papiers-core.tests = compile-only // flaky
 lloydmeta_enumeratum{

coordinator/src/main/scala/GitOps.scala

@@ -0,0 +1,102 @@
+import scala.annotation.tailrec
+import scala.concurrent.duration._
+
+object Git {
+  enum Revision:
+    case Branch(name: String)
+    case Tag(version: String)
+    case Commit(sha: String)
+    def stringValue = this match
+      case Branch(name) => name
+      case Tag(version) => version
+      case Commit(sha)  => sha
+  end Revision
+
+  def unshallowSinceDottyRelease(projectDir: os.Path): Unit =
+    // unshallow commits done after release Scala 3.0.0
+    os.proc("git", "fetch", s"--shallow-since=2021-05-13", "--quiet")
+      .call(cwd = projectDir, check = false)
+      .exitCode
+
+  def fetchTags(projectDir: os.Path): Unit =
+    os.proc("git", "fetch", "--tags", "--quiet")
+      .call(cwd = projectDir, check = false)
+      .exitCode
+
+  def checkout(
+      repoUrl: String,
+      projectName: String,
+      revision: Option[Revision],
+      depth: Option[Int]
+  ): Option[os.Path] = {
+    val branchOpt = revision.flatMap {
+      case Revision.Branch(name) => Some(s"--branch=$name")
+      case _                     => None
+    }
+    val depthOpt = depth.map(s"--depth=" + _)
+
+    @tailrec def tryClone[T](
+        retries: Int,
+        backoffSeconds: Int = 1
+    ): Option[os.Path] = {
+      val projectDir = os.temp.dir(prefix = s"repo-$projectName")
+      val proc = os
+        .proc(
+          "git",
+          "clone",
+          repoUrl,
+          projectDir,
+          "--quiet",
+          branchOpt,
+          depthOpt
+        )
+        .call(stderr = os.Pipe, check = false, timeout = 10.minutes.toMillis)
+
+      if proc.exitCode == 0 then Some(projectDir)
+      else if retries > 0 then
+        Console.err.println(
+          s"Failed to clone $repoUrl at revision ${revision}, backoff ${backoffSeconds}s"
+        )
+        proc.err.lines().foreach(Console.err.println)
+        Thread.sleep(backoffSeconds * 1000)
+        tryClone(retries - 1, (backoffSeconds * 2).min(60))
+      else
+        Console.err.println(
+          s"Failed to clone $repoUrl at revision ${revision}:"
+        )
+        proc.err.lines().foreach(Console.err.println)
+        None
+    }
+
+    def checkoutRevision(projectDir: os.Path): Boolean = revision match {
+      case None | Some(_: Revision.Branch)       => true // no need to checkout
+      case Some(Revision.Tag("master" | "main")) => true
+      case Some(revision: (Revision.Commit | Revision.Tag)) =>
+        val rev = revision match
+          case Revision.Commit(sha) =>
+            unshallowSinceDottyRelease(projectDir)
+            sha
+          case Revision.Tag(tag) =>
+            fetchTags(projectDir)
+            s"tags/$tag"
+
+        val proc = os
+          .proc("git", "checkout", rev, "--quiet")
+          .call(
+            cwd = projectDir,
+            check = false,
+            timeout = 15.seconds.toMillis,
+            mergeErrIntoOut = true
+          )
+        if (proc.exitCode != 0)
+          System.err.println(
+            s"Failed to checkout revision $revision: " + proc.out
+              .lines()
+              .mkString
+          )
+        proc.exitCode == 0
+    }
+
+    tryClone(retries = 10).filter(checkoutRevision(_))
+  }
+}

coordinator/src/main/scala/ProjectConfigDiscovery.scala

@@ -9,11 +9,12 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) {
   def apply(
       project: ProjectVersion,
       repoUrl: String,
-      tagOrRevision: Option[String]
+      revision: Option[Git.Revision]
   ): Option[ProjectBuildConfig] = {
     val name = project.showName
 
-    checkout(repoUrl, name, tagOrRevision)
+    Git
+      .checkout(repoUrl, name, revision, depth = Some(1))
       .flatMap { projectDir =>
         try {
           readProjectConfig(projectDir, repoUrl)
@@ -37,46 +38,6 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) {
       }
   }
 
-  private def checkout(
-      repoUrl: String,
-      projectName: String,
-      tagOrRevision: Option[String]
-  ): Option[os.Path] = {
-    @tailrec def retry[T](
-        retries: Int,
-        backoffSeconds: Int = 1
-    ): Option[os.Path] = {
-      val projectDir = os.temp.dir(prefix = s"repo-$projectName")
-      val proc = os
-        .proc(
-          "git",
-          "clone",
-          repoUrl,
-          projectDir,
-          "--quiet",
-          tagOrRevision.map("--branch=" + _).toList,
-          "--depth=1"
-        )
-        .call(stderr = os.Pipe, check = false)
-
-      if proc.exitCode == 0 then Some(projectDir)
-      else if retries > 0 then
-        Console.err.println(
-          s"Failed to checkout $repoUrl at revision ${tagOrRevision}, backoff ${backoffSeconds}s"
-        )
-        proc.err.lines().foreach(Console.err.println)
-        Thread.sleep(backoffSeconds * 1000)
-        retry(retries - 1, (backoffSeconds * 2).min(60))
-      else
-        Console.err.println(
-          s"Failed to checkout $repoUrl at revision ${tagOrRevision}:"
-        )
-        proc.err.lines().foreach(Console.err.println)
-        None
-    }
-    retry(retries = 10)
-  }
-
   private def githubWorkflows(projectDir: os.Path) = {
     val workflowsDir = projectDir / ".github" / "workflows"
     if !os.exists(workflowsDir) then Nil
@@ -167,7 +128,8 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) {
     // release is used by oracle-actions/setup-java@v1
     val JavaVersion = raw"(?:java-version|jdk|jvm|release):\s*(.*)".r
     val JavaVersionNumber = raw"$OptQuote(\d+)$OptQuote".r
-    val JavaVersionDistroVer = raw"$OptQuote(\w+)[@:]([\d\.]*[\w\-\_\.]*)$OptQuote".r
+    val JavaVersionDistroVer =
+      raw"$OptQuote(\w+)[@:]([\d\.]*[\w\-\_\.]*)$OptQuote".r
     val MatrixEntry = raw"(\w+):\s*\[(.*)\]".r
     // We can only supported this versions
     val allowedVersions = Seq(8, 11, 17, 21)
@@ -214,7 +176,7 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) {
         "scala3Version", // https://github.com/47degrees/fetch/blob/c4732a827816c58ce84013e9580120bdc3f64bc6/build.sbt#L10
         "Scala_3", // https://github.dev/kubukoz/sup/blob/644848c03173c726f19a40e6dd439b6905d42967/build.sbt#L10-L11
         "scala_3",
-        "`Scala-3`", 
+        "`Scala-3`",
         "`scala-3`" // https://github.com/rolang/dumbo/blob/7cc7f22ee45632b45bb1092418a3498ede8226da/build.sbt#L3
       )
       val Scala3VersionNamesAlt = matchEnclosed(

coordinator/src/main/scala/Scaladex.scala

@@ -5,8 +5,14 @@ import scala.concurrent.*
 object Scaladex {
   case class Pagination(current: Int, pageCount: Int, totalSize: Int)
   // releaseDate is always UTC zoned
-  case class ArtifactMetadata(version: String, releaseDate: java.time.OffsetDateTime)
-  case class ArtifactMetadataResponse(pagination: Pagination, items: List[ArtifactMetadata])
+  case class ArtifactMetadata(
+      version: String,
+      releaseDate: java.time.OffsetDateTime
+  )
+  case class ArtifactMetadataResponse(
+      pagination: Pagination,
+      items: List[ArtifactMetadata]
+  )
   case class ProjectSummary(
       groupId: String,
       artifacts: List[String], // List of artifacts with suffixes
@@ -20,20 +26,21 @@ object Scaladex {
       groupId: String,
       artifactId: String
   ): AsyncResponse[ArtifactMetadataResponse] = {
-    def tryFetch(backoffSeconds: Int): AsyncResponse[ArtifactMetadataResponse] = Future {
-      val response = requests.get(
-        url = s"$ScaladexUrl/api/artifacts/$groupId/$artifactId"
-      )
-      fromJson[ArtifactMetadataResponse](response.text())
-    }.recoverWith {
-      case err: org.jsoup.HttpStatusException
-          if err.getStatusCode == 503 && !Thread.interrupted() =>
-        Console.err.println(
-          s"Failed to fetch artifact metadata, Scaladex unavailable, retry with backoff ${backoffSeconds}s for $groupId:$artifactId"
+    def tryFetch(backoffSeconds: Int): AsyncResponse[ArtifactMetadataResponse] =
+      Future {
+        val response = requests.get(
+          url = s"$ScaladexUrl/api/artifacts/$groupId/$artifactId"
         )
-        SECONDS.sleep(backoffSeconds)
-        tryFetch((backoffSeconds * 2).min(60))
-    }
+        fromJson[ArtifactMetadataResponse](response.text())
+      }.recoverWith {
+        case err: org.jsoup.HttpStatusException
+            if err.getStatusCode == 503 && !Thread.interrupted() =>
+          Console.err.println(
+            s"Failed to fetch artifact metadata, Scaladex unavailable, retry with backoff ${backoffSeconds}s for $groupId:$artifactId"
+          )
+          SECONDS.sleep(backoffSeconds)
+          tryFetch((backoffSeconds * 2).min(60))
+      }
     tryFetch(1)
   }
 

coordinator/src/main/scala/build.scala

@@ -1,10 +1,10 @@
 //> using scala "3.2"
 //> using lib "com.novocode:junit-interface:0.11"
-//> using lib "org.jsoup:jsoup:1.16.1"
+//> using lib "org.jsoup:jsoup:1.16.2"
 //> using lib "org.json4s::json4s-native:4.0.6"
 //> using lib "org.json4s::json4s-ext:4.0.6"
 //> using lib "com.github.pureconfig::pureconfig-core:0.17.4"
-//> using lib "com.lihaoyi::os-lib:0.9.1"
+//> using lib "com.lihaoyi::os-lib:0.9.2"
 //> using lib "com.lihaoyi::requests:0.8.0"
 
 //> using resourceDir "../resources"