diff --git a/.github/actions/build-project/action.yaml b/.github/actions/build-project/action.yaml index d2d8b21b..3871ec6a 100644 --- a/.github/actions/build-project/action.yaml +++ b/.github/actions/build-project/action.yaml @@ -43,6 +43,10 @@ runs: ConfigFile=".github/workflows/buildConfig.json" DefaultJDK=11 javaVersion=$(jq -r ".\"${{ inputs.project-name }}\".config.java.version // ${DefaultJDK}" $ConfigFile) + if [[ $javaVersion -ge 21 && ! "$scalaVersion" = 3.3.* ]]; then + echo "Force Java 17, Java 21 is only supported since 3.3.x" + javaVersion=17 + fi echo "java-version=$javaVersion" >> $GITHUB_ENV echo "JavaVersion set to $javaVersion" @@ -119,6 +123,7 @@ runs: touch build-logs.txt build-summary.txt # Assume failure unless overwritten by a successful build echo 'failure' > build-status.txt + echo 'unknown' > build-tool.txt /build/build-revision.sh \ "$(config .project)" \ @@ -137,6 +142,7 @@ runs: mv build-logs.txt /opencb/ mv build-status.txt /opencb/ mv build-summary.txt /opencb/ + mv build-tool.txt /opencb/ - name: Check status id: check-status @@ -211,7 +217,8 @@ runs: "$(config .version)" \ "${{ inputs.scala-version }}" \ "${{ github.run_id }}" \ - "${{ steps.job-info.outputs.build-url }}" + "${{ steps.job-info.outputs.build-url }}" \ + "$(cat build-tool.txt)" if [ $? != 0 ]; then echo "::warning title=Indexing failure::Indexing results of ${{ inputs.project-name }} failed" fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index abcacb5e..ea2f95af 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,15 +37,7 @@ jobs: - uses: coursier/setup-action@v1 with: - apps: scala sbt mill:0.10.10 - - - name: Install scala-cli - run: | - curl -s --compressed "https://virtuslab.github.io/scala-cli-packages/KEY.gpg" | sudo apt-key add - - sudo curl -s --compressed -o /etc/apt/sources.list.d/scala_cli_packages.list "https://virtuslab.github.io/scala-cli-packages/debian/scala_cli_packages.list" - sudo apt-get update && \ - sudo apt-get install scala-cli && \ - scala-cli --version + apps: scala sbt mill:0.10.10 scala-cli:1.0.4 - name: Start minikube run: minikube start diff --git a/coordinator/.scalafmt.conf b/coordinator/.scalafmt.conf new file mode 100644 index 00000000..3d593eb9 --- /dev/null +++ b/coordinator/.scalafmt.conf @@ -0,0 +1,3 @@ +version = "3.7.14" +runner.dialect = scala3 +maxColumn=100 \ No newline at end of file diff --git a/coordinator/configs/projects-config.conf b/coordinator/configs/projects-config.conf index a302c831..d380a0ae 100644 --- a/coordinator/configs/projects-config.conf +++ b/coordinator/configs/projects-config.conf @@ -419,7 +419,8 @@ laserdisc-io_fs2-aws{ tests = compile-only sbt.commands = ["disableFatalWarnings"] } -lichess-org_lila.java.version = 17 +lichess-org_lila.java.version = 21 +lichess-org_playframework-lila.java.version = 21 lichess-org_scalachess.tests = compile-only // Deadlocks in containers linyxus_papiers-core.tests = compile-only // flaky lloydmeta_enumeratum{ diff --git a/coordinator/src/main/scala/GitOps.scala b/coordinator/src/main/scala/GitOps.scala new file mode 100644 index 00000000..64f70269 --- /dev/null +++ b/coordinator/src/main/scala/GitOps.scala @@ -0,0 +1,102 @@ +import scala.annotation.tailrec +import scala.concurrent.duration._ + +object Git { + enum Revision: + case Branch(name: String) + case Tag(version: String) + case Commit(sha: String) + def stringValue = this match + case Branch(name) => name + case Tag(version) => version + case Commit(sha) => sha + end Revision + + def unshallowSinceDottyRelease(projectDir: os.Path): Unit = + // unshallow commits done after release Scala 3.0.0 + os.proc("git", "fetch", s"--shallow-since=2021-05-13", "--quiet") + .call(cwd = projectDir, check = false) + .exitCode + + def fetchTags(projectDir: os.Path): Unit = + os.proc("git", "fetch", "--tags", "--quiet") + .call(cwd = projectDir, check = false) + .exitCode + + def checkout( + repoUrl: String, + projectName: String, + revision: Option[Revision], + depth: Option[Int] + ): Option[os.Path] = { + val branchOpt = revision.flatMap { + case Revision.Branch(name) => Some(s"--branch=$name") + case _ => None + } + val depthOpt = depth.map(s"--depth=" + _) + + @tailrec def tryClone[T]( + retries: Int, + backoffSeconds: Int = 1 + ): Option[os.Path] = { + val projectDir = os.temp.dir(prefix = s"repo-$projectName") + val proc = os + .proc( + "git", + "clone", + repoUrl, + projectDir, + "--quiet", + branchOpt, + depthOpt + ) + .call(stderr = os.Pipe, check = false, timeout = 10.minutes.toMillis) + + if proc.exitCode == 0 then Some(projectDir) + else if retries > 0 then + Console.err.println( + s"Failed to clone $repoUrl at revision ${revision}, backoff ${backoffSeconds}s" + ) + proc.err.lines().foreach(Console.err.println) + Thread.sleep(backoffSeconds * 1000) + tryClone(retries - 1, (backoffSeconds * 2).min(60)) + else + Console.err.println( + s"Failed to clone $repoUrl at revision ${revision}:" + ) + proc.err.lines().foreach(Console.err.println) + None + } + + def checkoutRevision(projectDir: os.Path): Boolean = revision match { + case None | Some(_: Revision.Branch) => true // no need to checkout + case Some(Revision.Tag("master" | "main")) => true + case Some(revision: (Revision.Commit | Revision.Tag)) => + val rev = revision match + case Revision.Commit(sha) => + unshallowSinceDottyRelease(projectDir) + sha + case Revision.Tag(tag) => + fetchTags(projectDir) + s"tags/$tag" + + val proc = os + .proc("git", "checkout", rev, "--quiet") + .call( + cwd = projectDir, + check = false, + timeout = 15.seconds.toMillis, + mergeErrIntoOut = true + ) + if (proc.exitCode != 0) + System.err.println( + s"Failed to checkout revision $revision: " + proc.out + .lines() + .mkString + ) + proc.exitCode == 0 + } + + tryClone(retries = 10).filter(checkoutRevision(_)) + } +} diff --git a/coordinator/src/main/scala/ProjectConfigDiscovery.scala b/coordinator/src/main/scala/ProjectConfigDiscovery.scala index fa29fd02..69f3a1bd 100644 --- a/coordinator/src/main/scala/ProjectConfigDiscovery.scala +++ b/coordinator/src/main/scala/ProjectConfigDiscovery.scala @@ -9,11 +9,12 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) { def apply( project: ProjectVersion, repoUrl: String, - tagOrRevision: Option[String] + revision: Option[Git.Revision] ): Option[ProjectBuildConfig] = { val name = project.showName - checkout(repoUrl, name, tagOrRevision) + Git + .checkout(repoUrl, name, revision, depth = Some(1)) .flatMap { projectDir => try { readProjectConfig(projectDir, repoUrl) @@ -37,46 +38,6 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) { } } - private def checkout( - repoUrl: String, - projectName: String, - tagOrRevision: Option[String] - ): Option[os.Path] = { - @tailrec def retry[T]( - retries: Int, - backoffSeconds: Int = 1 - ): Option[os.Path] = { - val projectDir = os.temp.dir(prefix = s"repo-$projectName") - val proc = os - .proc( - "git", - "clone", - repoUrl, - projectDir, - "--quiet", - tagOrRevision.map("--branch=" + _).toList, - "--depth=1" - ) - .call(stderr = os.Pipe, check = false) - - if proc.exitCode == 0 then Some(projectDir) - else if retries > 0 then - Console.err.println( - s"Failed to checkout $repoUrl at revision ${tagOrRevision}, backoff ${backoffSeconds}s" - ) - proc.err.lines().foreach(Console.err.println) - Thread.sleep(backoffSeconds * 1000) - retry(retries - 1, (backoffSeconds * 2).min(60)) - else - Console.err.println( - s"Failed to checkout $repoUrl at revision ${tagOrRevision}:" - ) - proc.err.lines().foreach(Console.err.println) - None - } - retry(retries = 10) - } - private def githubWorkflows(projectDir: os.Path) = { val workflowsDir = projectDir / ".github" / "workflows" if !os.exists(workflowsDir) then Nil @@ -167,7 +128,8 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) { // release is used by oracle-actions/setup-java@v1 val JavaVersion = raw"(?:java-version|jdk|jvm|release):\s*(.*)".r val JavaVersionNumber = raw"$OptQuote(\d+)$OptQuote".r - val JavaVersionDistroVer = raw"$OptQuote(\w+)[@:]([\d\.]*[\w\-\_\.]*)$OptQuote".r + val JavaVersionDistroVer = + raw"$OptQuote(\w+)[@:]([\d\.]*[\w\-\_\.]*)$OptQuote".r val MatrixEntry = raw"(\w+):\s*\[(.*)\]".r // We can only supported this versions val allowedVersions = Seq(8, 11, 17, 21) @@ -214,7 +176,7 @@ class ProjectConfigDiscovery(internalProjectConfigsPath: java.io.File) { "scala3Version", // https://github.com/47degrees/fetch/blob/c4732a827816c58ce84013e9580120bdc3f64bc6/build.sbt#L10 "Scala_3", // https://github.dev/kubukoz/sup/blob/644848c03173c726f19a40e6dd439b6905d42967/build.sbt#L10-L11 "scala_3", - "`Scala-3`", + "`Scala-3`", "`scala-3`" // https://github.com/rolang/dumbo/blob/7cc7f22ee45632b45bb1092418a3498ede8226da/build.sbt#L3 ) val Scala3VersionNamesAlt = matchEnclosed( diff --git a/coordinator/src/main/scala/Scaladex.scala b/coordinator/src/main/scala/Scaladex.scala index 42b1ef2e..fef497cd 100644 --- a/coordinator/src/main/scala/Scaladex.scala +++ b/coordinator/src/main/scala/Scaladex.scala @@ -5,8 +5,14 @@ import scala.concurrent.* object Scaladex { case class Pagination(current: Int, pageCount: Int, totalSize: Int) // releaseDate is always UTC zoned - case class ArtifactMetadata(version: String, releaseDate: java.time.OffsetDateTime) - case class ArtifactMetadataResponse(pagination: Pagination, items: List[ArtifactMetadata]) + case class ArtifactMetadata( + version: String, + releaseDate: java.time.OffsetDateTime + ) + case class ArtifactMetadataResponse( + pagination: Pagination, + items: List[ArtifactMetadata] + ) case class ProjectSummary( groupId: String, artifacts: List[String], // List of artifacts with suffixes @@ -20,20 +26,21 @@ object Scaladex { groupId: String, artifactId: String ): AsyncResponse[ArtifactMetadataResponse] = { - def tryFetch(backoffSeconds: Int): AsyncResponse[ArtifactMetadataResponse] = Future { - val response = requests.get( - url = s"$ScaladexUrl/api/artifacts/$groupId/$artifactId" - ) - fromJson[ArtifactMetadataResponse](response.text()) - }.recoverWith { - case err: org.jsoup.HttpStatusException - if err.getStatusCode == 503 && !Thread.interrupted() => - Console.err.println( - s"Failed to fetch artifact metadata, Scaladex unavailable, retry with backoff ${backoffSeconds}s for $groupId:$artifactId" + def tryFetch(backoffSeconds: Int): AsyncResponse[ArtifactMetadataResponse] = + Future { + val response = requests.get( + url = s"$ScaladexUrl/api/artifacts/$groupId/$artifactId" ) - SECONDS.sleep(backoffSeconds) - tryFetch((backoffSeconds * 2).min(60)) - } + fromJson[ArtifactMetadataResponse](response.text()) + }.recoverWith { + case err: org.jsoup.HttpStatusException + if err.getStatusCode == 503 && !Thread.interrupted() => + Console.err.println( + s"Failed to fetch artifact metadata, Scaladex unavailable, retry with backoff ${backoffSeconds}s for $groupId:$artifactId" + ) + SECONDS.sleep(backoffSeconds) + tryFetch((backoffSeconds * 2).min(60)) + } tryFetch(1) } diff --git a/coordinator/src/main/scala/build.scala b/coordinator/src/main/scala/build.scala index 42e77e7a..7354123f 100644 --- a/coordinator/src/main/scala/build.scala +++ b/coordinator/src/main/scala/build.scala @@ -1,10 +1,10 @@ //> using scala "3.2" //> using lib "com.novocode:junit-interface:0.11" -//> using lib "org.jsoup:jsoup:1.16.1" +//> using lib "org.jsoup:jsoup:1.16.2" //> using lib "org.json4s::json4s-native:4.0.6" //> using lib "org.json4s::json4s-ext:4.0.6" //> using lib "com.github.pureconfig::pureconfig-core:0.17.4" -//> using lib "com.lihaoyi::os-lib:0.9.1" +//> using lib "com.lihaoyi::os-lib:0.9.2" //> using lib "com.lihaoyi::requests:0.8.0" //> using resourceDir "../resources" diff --git a/coordinator/src/main/scala/buildPlan.scala b/coordinator/src/main/scala/buildPlan.scala index f96be83b..638bc6cd 100644 --- a/coordinator/src/main/scala/buildPlan.scala +++ b/coordinator/src/main/scala/buildPlan.scala @@ -14,6 +14,8 @@ import os.write import scala.collection.mutable import scala.collection.SortedMap import os.CommandResult +import java.time.LocalDate +import java.time.format.DateTimeFormatter class ConfigFiles(path: os.Path) { val projectsConfig: os.Path = path / "projects-config.conf" @@ -31,14 +33,29 @@ val ForReproducer = sys.props.contains("opencb.coordinator.reproducer-mode") maxProjectsInConfig: Int, maxProjectsInBuildPlan: Int, requiredProjects: Seq[Project], - configsPath: os.Path + configsPath: os.Path, + varargs: String* ) = { + val releaseCutOffDate = varargs.collectFirst { case s"--release-cutoff=${date}" => + Try(LocalDate.parse(date)).fold[Option[LocalDate]]( + ex => + System.err.println( + s"Failed to parse cutoff date: `$date` - ${ex.getMessage()}" + ) + None + , + parsed => + println(s"Would apply release cutoff date: $parsed") + Some(parsed) + ) + }.flatten given confFiles: ConfigFiles = ConfigFiles(configsPath) // Most of the time is spend in IO, though we can use higher parallelism val threadPool = new ForkJoinPool( Runtime.getRuntime().availableProcessors() * 4 ) - val customProjects = readNormalized(confFiles.customProjects).map(Project.load) + val customProjects = + readNormalized(confFiles.customProjects).map(Project.load) given ExecutionContext = ExecutionContext.fromExecutor(threadPool) val task = for { @@ -48,15 +65,22 @@ val ForReproducer = sys.props.contains("opencb.coordinator.reproducer-mode") maxProjectsCount = Option(maxProjectsInConfig).filter(_ >= 0), requiredProjects = requiredProjects, customProjects = customProjects, - filterPatterns = loadFilters + filterPatterns = loadFilters, + releaseCutOffDate = releaseCutOffDate + ) + _ = println( + s"Loaded dependency graph: ${dependencyGraph.projects.size} projects" + ) + fullBuildPlan <- makeDependenciesBasedBuildPlan( + dependencyGraph, + releaseCutOffDate ) - _ = println(s"Loaded dependency graph: ${dependencyGraph.projects.size} projects") - fullBuildPlan <- makeDependenciesBasedBuildPlan(dependencyGraph) _ = println("Generated build plan") } yield { // Build config if !ForReproducer then { - val configMap = SortedMap.from(fullBuildPlan.map(p => p.project.coordinates -> p)) + val configMap = + SortedMap.from(fullBuildPlan.map(p => p.project.coordinates -> p)) os.write.over( workflowsDir / "buildConfig.json", toJson(configMap, pretty = true), @@ -110,7 +134,10 @@ val ForReproducer = sys.props.contains("opencb.coordinator.reproducer-mode") } case class SplittedBuildPlan(projects: Array[ProjectBuildDef], index: Int = 0) -def splitBuildPlan(buildPlan: Array[ProjectBuildDef], limit: Int): List[SplittedBuildPlan] = { +def splitBuildPlan( + buildPlan: Array[ProjectBuildDef], + limit: Int +): List[SplittedBuildPlan] = { // Sort ascending by ammount of starts, required projects have highest priority buildPlan .sortBy { @@ -210,7 +237,10 @@ def buildPlanCommons(depGraph: DependencyGraph) = (topLevelData, fullInfo, projectsDeps) -def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using +def makeDependenciesBasedBuildPlan( + depGraph: DependencyGraph, + cutOffDate: Option[LocalDate] +)(using confFiles: ConfigFiles ): AsyncResponse[Array[ProjectBuildDef]] = val (topLevelData, fullInfo, projectsDeps) = buildPlanCommons(depGraph) @@ -220,7 +250,9 @@ def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using val replacementPattern = raw"(\S+)/(\S+) (\S+)/(\S+) ?(\S+)?".r val replacements = - if !os.exists(confFiles.replacedProjects) || os.isDir(confFiles.replacedProjects) + if !os.exists(confFiles.replacedProjects) || os.isDir( + confFiles.replacedProjects + ) then Map.empty else readNormalized(confFiles.replacedProjects).map { @@ -236,16 +268,70 @@ def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using def getRevision(project: Project) = val originalCoords = (project.org, project.name) - replacements.get(originalCoords).map(_._2).flatten + replacements + .get(originalCoords) + .map(_._2) + .flatten + .map(Git.Revision.Tag(_)) val projects = projectsDeps.keys.map(_.p).toList + def findCutOffCommit( + project: Project, + repoUrl: String, + cutOffDate: Option[LocalDate] + ): Option[String] = + for + cutOffDate <- cutOffDate + repoDir <- Git.checkout( + repoUrl, + project.name, + revision = None, + depth = Some(1) + ) + _ = Git.unshallowSinceDottyRelease(repoDir) + lastCommit <- os + .proc( + "git", + "--no-pager", + "log", + s"--before=${cutOffDate.format(DateTimeFormatter.ISO_DATE)}", + "--pretty=format:%H", + "--max-count=1" + ) + .call(cwd = repoDir, check = false, timeout = 5.minutes.toMillis) + .out + .lines() + .headOption + _ = os.remove.all(repoDir) // best-effort, it's tmp dir anyway + yield lastCommit.trim() + Future .traverse(projectsDeps.toList) { (project, deps) => Future { val repoUrl = projectRepoUrl(project.p) - val tag = - getRevision(project.p).orElse(findTag(repoUrl, project.v)) + val revision: Option[Git.Revision] = + getRevision(project.p) + .orElse( + findTag(repoUrl, project.v) + .tapEach(v => + println( + s"Would use tag: $v for ${project.p.coordinates} @ ${project.v}" + ) + ) + .headOption + .map(Git.Revision.Tag(_)) + ) + .orElse( + findCutOffCommit(project.p, repoUrl, cutOffDate) + .tapEach(v => + println( + s"Would use commit: $v for ${project.p.coordinates} @ ${project.v}" + ) + ) + .headOption + .map(Git.Revision.Commit(_)) + ) val self = project.p val dependencies = deps .map(_.p) @@ -259,7 +345,7 @@ def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using project = project.p, dependencies = dependencies.toArray, repoUrl = repoUrl, - revision = tag.getOrElse(""), + revision = revision.map(_.stringValue).getOrElse(""), version = project.v, targets = fullInfo(project.p).targets .map { @@ -267,20 +353,25 @@ def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using case t => stripScala3Suffix(t.id.asMvnStr) } .mkString(" "), - config = configDiscovery(project, repoUrl, tag) + config = configDiscovery(project, repoUrl, revision) ) } } .map(_.filter(_.project != DottyProject).toArray) -private def loadFilters(using confFiles: ConfigFiles): Seq[String] = readNormalized( - confFiles.filteredProjects -) -private def loadLongBuildingProjects()(using confFiles: ConfigFiles): Seq[Project] = +private def loadFilters(using confFiles: ConfigFiles): Seq[String] = + readNormalized( + confFiles.filteredProjects + ) +private def loadLongBuildingProjects()(using + confFiles: ConfigFiles +): Seq[Project] = readNormalized(confFiles.slowProjects).flatMap { case s"$org/$repo" => Some(Project(org, repo)) case malformed => - System.err.println(s"Malformed project long building project name: $malformed ") + System.err.println( + s"Malformed project long building project name: $malformed " + ) None } @@ -321,7 +412,9 @@ def splitIntoStages( } if currentStage.isEmpty then { def hasCyclicDependencies(p: ProjectBuildDef) = - p.dependencies.exists(deps.get(_).fold(false)(_.dependencies.contains(p.project))) + p.dependencies.exists( + deps.get(_).fold(false)(_.dependencies.contains(p.project)) + ) val cyclicDeps = newRemainings.filter(hasCyclicDependencies) if cyclicDeps.nonEmpty then { currentStage ++= cyclicDeps @@ -329,7 +422,7 @@ def splitIntoStages( cyclicDeps.foreach(v => println( s"Mitigated cyclic dependency in ${v.project} -> ${v.dependencies.toList - .filterNot(done.contains)}" + .filterNot(done.contains)}" ) ) } else { @@ -338,7 +431,8 @@ def splitIntoStages( .toSeq .sortBy { (depsCount, _) => depsCount } .head - def showCurrent = tieBreakers.map(_.project.coordinates).mkString(", ") + def showCurrent = + tieBreakers.map(_.project.coordinates).mkString(", ") System.err.println( s"Not found projects without already resolved dependencies, using [${tieBreakers.size}] projects with minimal dependency size=$minDeps : ${showCurrent}" ) @@ -499,8 +593,12 @@ def createGithubActionJob( }) println(" with:") println(" project-name: ${{ matrix.name }}") - println(" extra-scalac-options: ${{ inputs.extra-scalac-options }}") - println(" disabled-scalac-options: ${{ inputs.disabled-scalac-options }}") + println( + " extra-scalac-options: ${{ inputs.extra-scalac-options }}" + ) + println( + " disabled-scalac-options: ${{ inputs.disabled-scalac-options }}" + ) println(s" scala-version: $${{ $setupOutputs.scala-version }}") println(s" maven-repo-url: $${{ $setupOutputs.maven-repo-url }}") println(" elastic-user: ${{ secrets.OPENCB_ELASTIC_USER }}") diff --git a/coordinator/src/main/scala/cache.scala b/coordinator/src/main/scala/cache.scala index 5164634d..4652c891 100644 --- a/coordinator/src/main/scala/cache.scala +++ b/coordinator/src/main/scala/cache.scala @@ -56,7 +56,10 @@ given CacheDriver[String, Seq[StarredProject]] with v.map(_.serialize).mkString("\n") def load(data: String, k: String): Seq[StarredProject] = - data.linesIterator.map (Project.load).collect{case sp :StarredProject => sp}.toSeq + data.linesIterator + .map(Project.load) + .collect { case sp: StarredProject => sp } + .toSeq def dest(v: String): Path = dataPath.resolve(v) @@ -71,7 +74,8 @@ given CacheDriver[ModuleVersion, Target] with (Dep(v.id, "_") +: v.deps).map(DepOps.write).mkString("\n"): @unchecked def load(data: String, key: ModuleVersion): Target = - val Dep(id, _) +: deps = data.linesIterator.toSeq.map(DepOps.load): @unchecked + val Dep(id, _) +: deps = + data.linesIterator.toSeq.map(DepOps.load): @unchecked Target(id, deps) def dest(v: ModuleVersion): Path = diff --git a/coordinator/src/main/scala/core.scala b/coordinator/src/main/scala/core.scala index c01ee5be..e0979af7 100644 --- a/coordinator/src/main/scala/core.scala +++ b/coordinator/src/main/scala/core.scala @@ -38,7 +38,12 @@ case class ProjectVersion(p: Project, v: String) { def showName = p.show } -case class MvnMapping(name: String, version: String, mvn: String, deps: Seq[String]): +case class MvnMapping( + name: String, + version: String, + mvn: String, + deps: Seq[String] +): def show = (Seq(name, version, mvn) ++ deps).mkString(",") object MvnMapping: @@ -108,13 +113,19 @@ object ProjectBuildConfig { val empty = ProjectBuildConfig() } -case class SemVersion(major: Int, minor: Int, patch: Int, milestone: Option[String]) +case class SemVersion( + major: Int, + minor: Int, + patch: Int, + milestone: Option[String] +) given Conversion[String, SemVersion] = version => // There are multiple projects that don't follow standarnd naming convention, especially in snpashots // becouse of that it needs to be more flexible, e.g to handle: x.y-z-, x.y, x.y-milestone val parts = version.split('.').flatMap(_.split('-')).filter(_.nonEmpty) val versionNums = parts.take(3).takeWhile(_.forall(_.isDigit)) - def versionPart(idx: Int) = versionNums.lift(idx).flatMap(_.toIntOption).getOrElse(0) + def versionPart(idx: Int) = + versionNums.lift(idx).flatMap(_.toIntOption).getOrElse(0) val milestone = Some(parts.drop(versionNums.size)) .filter(_.nonEmpty) .map(_.mkString("-")) diff --git a/coordinator/src/main/scala/deps.scala b/coordinator/src/main/scala/deps.scala index 7f901326..0a18db34 100644 --- a/coordinator/src/main/scala/deps.scala +++ b/coordinator/src/main/scala/deps.scala @@ -4,7 +4,7 @@ import java.nio.file._ import scala.sys.process._ import scala.concurrent.* import scala.concurrent.duration.* -import java.time.OffsetDateTime +import java.time.{OffsetDateTime, LocalDate} import java.util.concurrent.TimeUnit.SECONDS import java.net.SocketTimeoutException import java.net.UnknownHostException @@ -33,7 +33,9 @@ def loadProjects(scalaBinaryVersion: String): Seq[StarredProject] = val texts = e.select("h4").get(0).text().split("/") val stars = e.select(".stats [title=Stars]").asScala.map(_.text) Option.unless(texts.isEmpty || stars.isEmpty) { - StarredProject(texts.head, texts.drop(1).mkString("/"))(stars.head.toInt) + StarredProject(texts.head, texts.drop(1).mkString("/"))( + stars.head.toInt + ) } } LazyList @@ -50,7 +52,10 @@ enum CandidateProject: case BuildSelected(project: Project, mvs: Seq[ModuleInVersion]) case class ProjectModules(project: Project, mvs: Seq[ModuleInVersion]) -def loadScaladexProject(scalaBinaryVersion: String)( +def loadScaladexProject( + scalaBinaryVersion: String, + releaseCutOffDate: Option[LocalDate] +)( project: Project ): AsyncResponse[ProjectModules] = import util.* @@ -82,6 +87,10 @@ def loadScaladexProject(scalaBinaryVersion: String)( // Order versions based on their release date, it should be more stable in case of hash-based pre-releases // Previous approach with sorting SemVersion was not stable and could lead to runtime erros (due to not transitive order of elements) val versions = response.items + .filter(v => + releaseCutOffDate + .forall(_.isAfter(v.releaseDate.toLocalDate())) + ) .tapEach(v => releaseDates += v.version -> v.releaseDate) .map(_.version) artifact -> versions @@ -90,7 +99,9 @@ def loadScaladexProject(scalaBinaryVersion: String)( .map(_.toMap) orderedVersions = projectSummary.versions .flatMap(v => releaseDates.get(v).map(VersionRelease(v, _))) - .sortBy(_.releaseDate)(using summon[Ordering[OffsetDateTime]].reverse) + .sortBy(_.releaseDate)(using + summon[Ordering[OffsetDateTime]].reverse + ) .map(_.version) yield for version <- orderedVersions yield ModuleInVersion( @@ -194,16 +205,19 @@ def loadDepenenecyGraph( maxProjectsCount: Option[Int] = None, requiredProjects: Seq[Project] = Nil, customProjects: Seq[Project] = Nil, - filterPatterns: Seq[String] = Nil + filterPatterns: Seq[String] = Nil, + releaseCutOffDate: Option[LocalDate] = None ): AsyncResponse[DependencyGraph] = val patterns = filterPatterns.map(_.r) def loadProject(p: Project): AsyncResponse[CandidateProject] = if customProjects.contains(p) then Future.successful(CandidateProject.BuildAll(p)) else cachedAsync { (p: Project) => - loadScaladexProject(scalaBinaryVersion)(p) + loadScaladexProject(scalaBinaryVersion, releaseCutOffDate)(p) .map(projectModulesFilter(patterns)) - }(p).map { case ProjectModules(project, mvs) => CandidateProject.BuildSelected(project, mvs) } + }(p).map { case ProjectModules(project, mvs) => + CandidateProject.BuildSelected(project, mvs) + } val required = LazyList .from(requiredProjects) @@ -231,7 +245,9 @@ def loadDepenenecyGraph( mvnInfo <- project match case CandidateProject.BuildAll(project) => - Future.successful(Some(LoadedProject(project, "HEAD", Seq(Target.BuildAll)))) + Future.successful( + Some(LoadedProject(project, "HEAD", Seq(Target.BuildAll))) + ) case candidate @ CandidateProject.BuildSelected(project, mvs) => if mvs.isEmpty then Future.successful(None) @@ -243,7 +259,9 @@ def loadDepenenecyGraph( } .recover { case ex: org.jsoup.HttpStatusException if ex.getStatusCode() == 404 => - System.err.println(s"Missing Maven info: ${ex.getUrl()}") + System.err.println( + s"Missing Maven info: ${ex.getUrl()}" + ) None } yield mvnInfo @@ -253,7 +271,9 @@ def loadDepenenecyGraph( load( required #::: optional( from = 0, - limit = maxProjectsCount.map(_ - requiredProjects.length - customProjects.length).map(_ max 0) + limit = maxProjectsCount + .map(_ - requiredProjects.length - customProjects.length) + .map(_ max 0) ) ).flatMap { loaded => val available = loaded.flatten @@ -264,8 +284,11 @@ def loadDepenenecyGraph( else { val continueFrom = loaded.size - required.size // Load '10 < 1/2n < 50' more projects then number of remaining slots to filter out possibly empty entries - val toLoad = remainingSlots + (remainingSlots * 0.5).toInt.max(10).min(50) - println(s"Filling remaining ${remainingSlots} slots, trying to load $toLoad next projects") + val toLoad = + remainingSlots + (remainingSlots * 0.5).toInt.max(10).min(50) + println( + s"Filling remaining ${remainingSlots} slots, trying to load $toLoad next projects" + ) load(optional(from = continueFrom, limit = Some(remainingSlots))) .map(available ++ _.flatten.take(remainingSlots)) } diff --git a/project-builder/build-revision.sh b/project-builder/build-revision.sh index 1f789b56..5467ba41 100755 --- a/project-builder/build-revision.sh +++ b/project-builder/build-revision.sh @@ -22,19 +22,23 @@ scriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export OPENCB_SCRIPT_DIR=$scriptDir $scriptDir/checkout.sh "$repoUrl" "$rev" repo +buildToolFile="build-tool.txt" if [ -f "repo/mill" ] || [ -f "repo/build.sc" ]; then echo "Mill project found: ${isMillProject}" + echo "mill" > $buildToolFile $scriptDir/mill/prepare-project.sh "$project" repo "$scalaVersion" "$version" "$projectConfig" $scriptDir/mill/build.sh repo "$scalaVersion" "$version" "$targets" "$mvnRepoUrl" "$projectConfig" "$extraScalacOptions" "$disabledScalacOption" elif [ -f "repo/build.sbt" ]; then echo "sbt project found: ${isSbtProject}" + echo "sbt" > $buildToolFile $scriptDir/sbt/prepare-project.sh "$project" repo "$enforcedSbtVersion" "$scalaVersion" "$projectConfig" $scriptDir/sbt/build.sh repo "$scalaVersion" "$version" "$targets" "$mvnRepoUrl" "$projectConfig" "$extraScalacOptions" "$disabledScalacOption" else echo "Not found sbt or mill build files, assuming scala-cli project" ls -l repo/ + echo "scala-cli" > $buildToolFile scala-cli clean $scriptDir/scala-cli/ scala-cli clean repo scala-cli $scriptDir/scala-cli/build.scala -- repo "$scalaVersion" "$projectConfig" "$mvnRepoUrl" diff --git a/project-builder/checkout.sh b/project-builder/checkout.sh index ad058f2a..79457f5c 100755 --- a/project-builder/checkout.sh +++ b/project-builder/checkout.sh @@ -20,4 +20,4 @@ if [ -n "$rev" ]; then branch="-b $rev" fi git clone --quiet --recurse-submodules "$repo" "$repoDir" $branch || - ( git clone --quiet --recurse-submodules "$repo" "$repoDir" && cd $repoDir && git checkout $rev ) + ( git clone --quiet --recurse-submodules "$repo" "$repoDir" && cd $repoDir && git fetch --shallow-since=2021-05-13 && git fetch --tags && git checkout $rev ) diff --git a/project-builder/feed-elastic.sh b/project-builder/feed-elastic.sh index a8505500..9467c2b2 100755 --- a/project-builder/feed-elastic.sh +++ b/project-builder/feed-elastic.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -if [ $# -ne 10 ]; then +if [ $# -ne 11 ]; then echo "Wrong number of script arguments, got $#, expected 7" exit 1 fi @@ -16,6 +16,7 @@ version="$7" scalaVersion="$8" buildId="$9" buildUrl="${10}" +buildTool="${11}" buildSummary="$(cat ${buildSummaryFile})" if [[ -z "${buildSummary}" ]]; then @@ -30,9 +31,10 @@ json=$(jq -n \ --arg scVer "$scalaVersion" \ --arg buildId "$buildId" \ --arg buildURL "$buildUrl" \ + --arg buildTool "$buildTool" \ --argjson sum "$buildSummary" \ --rawfile logs "$logsFile" \ - '{projectName: $pn, version: $ver, scalaVersion: $scVer, status: $res, timestamp: $ts, buildId: $buildId, buildURL: $buildURL, summary: $sum, logs: $logs}') + '{projectName: $pn, version: $ver, scalaVersion: $scVer, status: $res, timestamp: $ts, buildId: $buildId, buildURL: $buildURL, buildTool: $buildTool, summary: $sum, logs: $logs}') jsonFile=$(mktemp /tmp/feed-elastic-tmp.XXXXXX) diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index 28760179..465bd859 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -10,9 +10,9 @@ cd $scriptDir/../cli testNamespace=scala3-community-build-test cliRunCmd="run scb-cli.scala --jvm=11 --java-prop communitybuild.version=test --java-prop communitybuild.forced-java-version=11 --java-prop communitybuild.local.dir=$scriptDir/.. -- " commonOpts="--namespace=$testNamespace --noRedirectLogs" -sbtProject="typelevel/shapeless-3 --revision=v3.1.0" +sbtProject="typelevel/shapeless-3 --revision=v3.3.0" millProject="com-lihaoyi/os-lib --revision=0.8.1" -scalaVersion=3.1.1 +scalaVersion=3.3.0 #echo "Test sbt custom build in minikube" #scala-cli $cliRunCmd run $sbtProject $scalaVersion $commonOpts