From 17083077f33890c91063e274c80ec28a48d2c4a7 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 2 Oct 2024 22:21:49 +0800 Subject: [PATCH] Fix `PromptLogger` 60s delay, improve test grouping prompt lines, breakup large `scalalib` test suites (#3649) * Break up some of the long-poll tests in scalalib.test into small classes, in preparation for `testForkGrouping` * Fix issue with `PromptLogger` waiting an extra 60s when paused before exiting when calling `shutdown` * Improve the prompt line message for the test groups to show a collapsed list of test class names (which will get further collapsed by the prompt as necessary) --- main/util/src/mill/util/PromptLogger.scala | 3 +- .../src/mill/scalalib/TestModuleUtil.scala | 43 ++- .../src/mill/scalalib/AssemblyExeTests.scala | 45 +++ .../mill/scalalib/AssemblyNoExeTests.scala | 24 ++ .../src/mill/scalalib/AssemblyTestUtils.scala | 66 ++++ .../src/mill/scalalib/AssemblyTests.scala | 117 ------- .../scalalib/ScalaAssemblyAppendTests.scala | 103 ++++++ .../scalalib/ScalaAssemblyExcludeTests.scala | 117 +++++++ .../scalalib/ScalaAssemblyTestUtils.scala | 107 ++++++ .../mill/scalalib/ScalaAssemblyTests.scala | 307 +----------------- .../mill/scalalib/TestModuleUtilTests.scala | 104 ++++++ .../scalalib/TestRunnerScalatestTests.scala | 59 ++++ .../mill/scalalib/TestRunnerTestUtils.scala | 136 ++++++++ .../src/mill/scalalib/TestRunnerTests.scala | 296 ----------------- .../mill/scalalib/TestRunnerUtestTests.scala | 121 +++++++ .../scalalib/TestRunnerZiotestTests.scala | 32 ++ 16 files changed, 958 insertions(+), 722 deletions(-) create mode 100644 scalalib/test/src/mill/scalalib/AssemblyExeTests.scala create mode 100644 scalalib/test/src/mill/scalalib/AssemblyNoExeTests.scala create mode 100644 scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala delete mode 100644 scalalib/test/src/mill/scalalib/AssemblyTests.scala create mode 100644 scalalib/test/src/mill/scalalib/ScalaAssemblyAppendTests.scala create mode 100644 scalalib/test/src/mill/scalalib/ScalaAssemblyExcludeTests.scala create mode 100644 scalalib/test/src/mill/scalalib/ScalaAssemblyTestUtils.scala create mode 100644 scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala create mode 100644 scalalib/test/src/mill/scalalib/TestRunnerTestUtils.scala delete mode 100644 scalalib/test/src/mill/scalalib/TestRunnerTests.scala create mode 100644 scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala create mode 100644 scalalib/test/src/mill/scalalib/TestRunnerZiotestTests.scala diff --git a/main/util/src/mill/util/PromptLogger.scala b/main/util/src/mill/util/PromptLogger.scala index 0591fb5cf4c..daa65b0473b 100644 --- a/main/util/src/mill/util/PromptLogger.scala +++ b/main/util/src/mill/util/PromptLogger.scala @@ -91,7 +91,7 @@ private[mill] class PromptLogger( } def ticker(s: String): Unit = () - override def setPromptDetail(key: Seq[String], s: String): Unit = { + override def setPromptDetail(key: Seq[String], s: String): Unit = synchronized { state.updateDetail(key, s) } @@ -138,6 +138,7 @@ private[mill] class PromptLogger( else { pauseNoticed = false paused = true + promptUpdaterThread.interrupt() try { // After the prompt gets paused, wait until the `promptUpdaterThread` marks // `pauseNoticed = true`, so we can be sure it's done printing out prompt updates for diff --git a/scalalib/src/mill/scalalib/TestModuleUtil.scala b/scalalib/src/mill/scalalib/TestModuleUtil.scala index 3db401c3259..ef62c2f7707 100644 --- a/scalalib/src/mill/scalalib/TestModuleUtil.scala +++ b/scalalib/src/mill/scalalib/TestModuleUtil.scala @@ -42,7 +42,7 @@ private[scalalib] object TestModuleUtil { EnvVars.MILL_WORKSPACE_ROOT -> T.workspace.toString ) - def runTestSubprocess(selectors2: Seq[String], base: os.Path) = { + def runTestRunnerSubprocess(selectors2: Seq[String], base: os.Path) = { val outputPath = base / "out.json" val testArgs = TestArgs( framework = testFramework, @@ -93,20 +93,23 @@ private[scalalib] object TestModuleUtil { val subprocessResult: Either[String, (String, Seq[TestResult])] = filteredClassLists match { // When no tests at all are discovered, run at least one test JVM // process to go through the test framework setup/teardown logic - case Nil => runTestSubprocess(Nil, T.dest) - case Seq(singleTestClassList) => runTestSubprocess(singleTestClassList, T.dest) + case Nil => runTestRunnerSubprocess(Nil, T.dest) + case Seq(singleTestClassList) => runTestRunnerSubprocess(singleTestClassList, T.dest) case multipleTestClassLists => - val hasMultiClassGroup = multipleTestClassLists.exists(_.length > 1) val futures = multipleTestClassLists.zipWithIndex.map { case (testClassList, i) => - val groupLabel = testClassList match { - case Seq(single) => - if (hasMultiClassGroup) s"group-$i-$single" - else single - case multiple => s"group-$i" + val groupPromptMessage = testClassList match { + case Seq(single) => single + case multiple => + collapseTestClassNames(multiple).mkString(", ") + s", ${multiple.length} suites" } - T.fork.async(T.dest / groupLabel, "" + i, groupLabel) { - (groupLabel, runTestSubprocess(testClassList, T.dest / groupLabel)) + val folderName = testClassList match { + case Seq(single) => single + case multiple => s"group-$i-${multiple.head}" + } + + T.fork.async(T.dest / folderName, "" + i, groupPromptMessage) { + (folderName, runTestRunnerSubprocess(testClassList, T.dest / folderName)) } } @@ -295,4 +298,22 @@ private[scalalib] object TestModuleUtil { case _ => None } } + + /** + * Shorten the long list of fully qualified class names by truncating + * repetitive segments so we can see more stuff on a single line + */ + def collapseTestClassNames(names0: Seq[String]): Seq[String] = { + val names = names0.sorted + Seq(names.head) ++ names.sliding(2).map { + case Seq(prev, next) => + val prevSegments = prev.split('.') + val nextSegments = next.split('.') + + nextSegments + .zipWithIndex + .map { case (s, i) => if (prevSegments.lift(i).contains(s)) s.head else s } + .mkString(".") + } + } } diff --git a/scalalib/test/src/mill/scalalib/AssemblyExeTests.scala b/scalalib/test/src/mill/scalalib/AssemblyExeTests.scala new file mode 100644 index 00000000000..bb1b9b07b20 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/AssemblyExeTests.scala @@ -0,0 +1,45 @@ +package mill.scalalib + +import scala.util.Properties +import mill.api.Result +import mill.testkit.{UnitTester, TestBaseModule} +import utest._ + +// Ensure the assembly is runnable, even if we have assembled lots of dependencies into it +// Reproduction of issues: +// - https://github.com/com-lihaoyi/mill/issues/528 +// - https://github.com/com-lihaoyi/mill/issues/2650 + +object AssemblyExeTests extends TestSuite with AssemblyTestUtils { + + def tests: Tests = Tests { + test("Assembly") { + test("exe") { + test("small") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => + val Right(result) = eval(TestCase.exe.small.assembly) + val originalPath = result.value.path + val resolvedPath = + if (Properties.isWin) { + val winPath = originalPath / os.up / s"${originalPath.last}.bat" + os.copy(originalPath, winPath) + winPath + } else originalPath + runAssembly(resolvedPath, TestCase.millSourcePath, checkExe = true) + } + + test("large-should-fail") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => + val Left(Result.Failure(msg, Some(res))) = eval(TestCase.exe.large.assembly) + val expectedMsg = + """The created assembly jar contains more than 65535 ZIP entries. + |JARs of that size are known to not work correctly with a prepended shell script. + |Either reduce the entries count of the assembly or disable the prepended shell script with: + | + | def prependShellScript = "" + |""".stripMargin + assert(msg == expectedMsg) + + } + } + } + } +} diff --git a/scalalib/test/src/mill/scalalib/AssemblyNoExeTests.scala b/scalalib/test/src/mill/scalalib/AssemblyNoExeTests.scala new file mode 100644 index 00000000000..4b58bb0687b --- /dev/null +++ b/scalalib/test/src/mill/scalalib/AssemblyNoExeTests.scala @@ -0,0 +1,24 @@ +package mill.scalalib + +import mill.testkit.{UnitTester, TestBaseModule} +import utest._ + +object AssemblyNoExeTests extends TestSuite with AssemblyTestUtils { + + def tests: Tests = Tests { + test("Assembly") { + test("noExe") { + test("small") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => + val Right(result) = eval(TestCase.noExe.small.assembly) + runAssembly(result.value.path, TestCase.millSourcePath) + + } + test("large") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => + val Right(result) = eval(TestCase.noExe.large.assembly) + runAssembly(result.value.path, TestCase.millSourcePath) + + } + } + } + } +} diff --git a/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala b/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala new file mode 100644 index 00000000000..c6f1ecd16f2 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala @@ -0,0 +1,66 @@ +package mill.scalalib + +import mill._ +import mill.util.Jvm +import mill.testkit.{UnitTester, TestBaseModule} + +trait AssemblyTestUtils { + + object TestCase extends TestBaseModule { + trait Setup extends ScalaModule { + def scalaVersion = "2.13.11" + + def sources = Task.Sources(T.workspace / "src") + + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"com.lihaoyi::scalatags:0.8.2", + ivy"com.lihaoyi::mainargs:0.4.0", + ivy"org.apache.avro:avro:1.11.1" + ) + } + + trait ExtraDeps extends ScalaModule { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"dev.zio::zio:2.0.15", + ivy"org.typelevel::cats-core:2.9.0", + ivy"org.apache.spark::spark-core:3.4.0", + ivy"dev.zio::zio-metrics-connectors:2.0.8", + ivy"dev.zio::zio-http:3.0.0-RC2" + ) + } + + object noExe extends Module { + object small extends Setup { + override def prependShellScript: T[String] = "" + } + + object large extends Setup with ExtraDeps { + override def prependShellScript: T[String] = "" + } + } + + object exe extends Module { + object small extends Setup + + object large extends Setup with ExtraDeps + } + + } + + val sources = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "assembly" + def runAssembly(file: os.Path, wd: os.Path, checkExe: Boolean = false): Unit = { + println(s"File size: ${os.stat(file).size}") + Jvm.runSubprocess( + commandArgs = Seq(Jvm.javaExe, "-jar", file.toString(), "--text", "tutu"), + envArgs = Map.empty[String, String], + workingDir = wd + ) + if (checkExe) { + Jvm.runSubprocess( + commandArgs = Seq(file.toString(), "--text", "tutu"), + envArgs = Map.empty[String, String], + workingDir = wd + ) + } + } +} diff --git a/scalalib/test/src/mill/scalalib/AssemblyTests.scala b/scalalib/test/src/mill/scalalib/AssemblyTests.scala deleted file mode 100644 index 72af5839fda..00000000000 --- a/scalalib/test/src/mill/scalalib/AssemblyTests.scala +++ /dev/null @@ -1,117 +0,0 @@ -package mill.scalalib - -import scala.util.Properties -import mill._ -import mill.api.Result -import mill.eval.Evaluator -import mill.util.Jvm -import mill.testkit.{UnitTester, TestBaseModule} -import utest._ -import utest.framework.TestPath - -import java.io.PrintStream - -// Ensure the assembly is runnable, even if we have assembled lots of dependencies into it -// Reproduction of issues: -// - https://github.com/com-lihaoyi/mill/issues/528 -// - https://github.com/com-lihaoyi/mill/issues/2650 - -object AssemblyTests extends TestSuite { - - object TestCase extends TestBaseModule { - trait Setup extends ScalaModule { - def scalaVersion = "2.13.11" - def sources = Task.Sources(T.workspace / "src") - def ivyDeps = super.ivyDeps() ++ Agg( - ivy"com.lihaoyi::scalatags:0.8.2", - ivy"com.lihaoyi::mainargs:0.4.0", - ivy"org.apache.avro:avro:1.11.1" - ) - } - trait ExtraDeps extends ScalaModule { - def ivyDeps = super.ivyDeps() ++ Agg( - ivy"dev.zio::zio:2.0.15", - ivy"org.typelevel::cats-core:2.9.0", - ivy"org.apache.spark::spark-core:3.4.0", - ivy"dev.zio::zio-metrics-connectors:2.0.8", - ivy"dev.zio::zio-http:3.0.0-RC2" - ) - } - - object noExe extends Module { - object small extends Setup { - override def prependShellScript: T[String] = "" - } - object large extends Setup with ExtraDeps { - override def prependShellScript: T[String] = "" - } - } - - object exe extends Module { - object small extends Setup - object large extends Setup with ExtraDeps - } - - } - - val sources = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "assembly" - - def runAssembly(file: os.Path, wd: os.Path, checkExe: Boolean = false): Unit = { - println(s"File size: ${os.stat(file).size}") - Jvm.runSubprocess( - commandArgs = Seq(Jvm.javaExe, "-jar", file.toString(), "--text", "tutu"), - envArgs = Map.empty[String, String], - workingDir = wd - ) - if (checkExe) { - Jvm.runSubprocess( - commandArgs = Seq(file.toString(), "--text", "tutu"), - envArgs = Map.empty[String, String], - workingDir = wd - ) - } - } - - def tests: Tests = Tests { - test("Assembly") { - test("noExe") { - test("small") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => - val Right(result) = eval(TestCase.noExe.small.assembly) - runAssembly(result.value.path, TestCase.millSourcePath) - - } - test("large") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => - val Right(result) = eval(TestCase.noExe.large.assembly) - runAssembly(result.value.path, TestCase.millSourcePath) - - } - } - test("exe") { - test("small") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => - val Right(result) = eval(TestCase.exe.small.assembly) - val originalPath = result.value.path - val resolvedPath = - if (Properties.isWin) { - val winPath = originalPath / os.up / s"${originalPath.last}.bat" - os.copy(originalPath, winPath) - winPath - } else originalPath - runAssembly(resolvedPath, TestCase.millSourcePath, checkExe = true) - } - - test("large-should-fail") - UnitTester(TestCase, sourceRoot = sources).scoped { eval => - val Left(Result.Failure(msg, Some(res))) = eval(TestCase.exe.large.assembly) - val expectedMsg = - """The created assembly jar contains more than 65535 ZIP entries. - |JARs of that size are known to not work correctly with a prepended shell script. - |Either reduce the entries count of the assembly or disable the prepended shell script with: - | - | def prependShellScript = "" - |""".stripMargin - assert(msg == expectedMsg) - - } - } - } - } -} diff --git a/scalalib/test/src/mill/scalalib/ScalaAssemblyAppendTests.scala b/scalalib/test/src/mill/scalalib/ScalaAssemblyAppendTests.scala new file mode 100644 index 00000000000..b24af31e6d4 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/ScalaAssemblyAppendTests.scala @@ -0,0 +1,103 @@ +package mill.scalalib + +import mill._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ + +import java.util.jar.JarFile +import scala.util.Using +import HelloWorldTests._ + +object ScalaAssemblyAppendTests extends TestSuite with ScalaAssemblyTestUtils { + def tests: Tests = Tests { + def checkAppend[M <: mill.testkit.TestBaseModule](module: M, target: Target[PathRef]) = + UnitTester(module, resourcePath).scoped { eval => + val Right(result) = eval.apply(target) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + // akka modules configs are present + referenceContent.contains("akka-http Reference Config File"), + referenceContent.contains("akka-http-core Reference Config File"), + referenceContent.contains("Akka Actor Reference Config File"), + referenceContent.contains("Akka Stream Reference Config File"), + // our application config is present too + referenceContent.contains("My application Reference Config File"), + referenceContent.contains( + """akka.http.client.user-agent-header="hello-world-client"""" + ) + ) + } + } + + def checkAppendMulti[M <: mill.testkit.TestBaseModule]( + module: M, + target: Target[PathRef] + ): Unit = UnitTester( + module, + sourceRoot = helloWorldMultiResourcePath + ).scoped { eval => + val Right(result) = eval.apply(target) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + // reference config from core module + referenceContent.contains("Core Reference Config File"), + // reference config from model module + referenceContent.contains("Model Reference Config File"), + // concatenated content + referenceContent.contains("bar.baz=hello"), + referenceContent.contains("foo.bar=2") + ) + } + } + + def checkAppendWithSeparator[M <: mill.testkit.TestBaseModule]( + module: M, + target: Target[PathRef] + ): Unit = UnitTester( + module, + sourceRoot = helloWorldMultiResourcePath + ).scoped { eval => + val Right(result) = eval.apply(target) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(jarEntries(jarFile).contains("without-new-line.conf")) + + val result = readFileFromJar(jarFile, "without-new-line.conf").split('\n').toSet + val expected = Set("without-new-line.first=first", "without-new-line.second=second") + assert(result == expected) + } + } + + test("appendWithDeps") - checkAppend( + HelloWorldAkkaHttpAppend, + HelloWorldAkkaHttpAppend.core.assembly + ) + test("appendMultiModule") - checkAppendMulti( + HelloWorldMultiAppend, + HelloWorldMultiAppend.core.assembly + ) + test("appendPatternWithDeps") - checkAppend( + HelloWorldAkkaHttpAppendPattern, + HelloWorldAkkaHttpAppendPattern.core.assembly + ) + test("appendPatternMultiModule") - checkAppendMulti( + HelloWorldMultiAppendPattern, + HelloWorldMultiAppendPattern.core.assembly + ) + test("appendPatternMultiModuleWithSeparator") - checkAppendWithSeparator( + HelloWorldMultiAppendByPatternWithSeparator, + HelloWorldMultiAppendByPatternWithSeparator.core.assembly + ) + + } +} diff --git a/scalalib/test/src/mill/scalalib/ScalaAssemblyExcludeTests.scala b/scalalib/test/src/mill/scalalib/ScalaAssemblyExcludeTests.scala new file mode 100644 index 00000000000..30520d906d0 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/ScalaAssemblyExcludeTests.scala @@ -0,0 +1,117 @@ +package mill.scalalib + +import mill._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ + +import java.util.jar.JarFile +import scala.util.Using +import HelloWorldTests._ +object ScalaAssemblyExcludeTests extends TestSuite with ScalaAssemblyTestUtils { + def tests: Tests = Tests { + def checkExclude[M <: mill.testkit.TestBaseModule]( + module: M, + target: Target[PathRef], + resourcePath: os.Path = resourcePath + ) = UnitTester(module, resourcePath).scoped { eval => + val Right(result) = eval.apply(target) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(!jarEntries(jarFile).contains("reference.conf")) + } + } + + test("excludeWithDeps") - checkExclude( + HelloWorldAkkaHttpExclude, + HelloWorldAkkaHttpExclude.core.assembly + ) + test("excludeMultiModule") - checkExclude( + HelloWorldMultiExclude, + HelloWorldMultiExclude.core.assembly, + resourcePath = helloWorldMultiResourcePath + ) + test("excludePatternWithDeps") - checkExclude( + HelloWorldAkkaHttpExcludePattern, + HelloWorldAkkaHttpExcludePattern.core.assembly + ) + test("excludePatternMultiModule") - checkExclude( + HelloWorldMultiExcludePattern, + HelloWorldMultiExcludePattern.core.assembly, + resourcePath = helloWorldMultiResourcePath + ) + + def checkRelocate[M <: mill.testkit.TestBaseModule]( + module: M, + target: Target[PathRef], + resourcePath: os.Path = resourcePath + ) = UnitTester(module, resourcePath).scoped { eval => + val Right(result) = eval.apply(target) + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(!jarEntries(jarFile).contains("akka/http/scaladsl/model/HttpEntity.class")) + assert( + jarEntries(jarFile).contains("shaded/akka/http/scaladsl/model/HttpEntity.class") + ) + } + } + + test("relocate") { + test("withDeps") - checkRelocate( + HelloWorldAkkaHttpRelocate, + HelloWorldAkkaHttpRelocate.core.assembly + ) + + test("run") - UnitTester( + HelloWorldAkkaHttpRelocate, + sourceRoot = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-world-deps" + ).scoped { eval => + val Right(result) = eval.apply(HelloWorldAkkaHttpRelocate.core.runMain("Main")) + assert(result.evalCount > 0) + } + } + + test("writeDownstreamWhenNoRule") { + test("withDeps") - UnitTester(HelloWorldAkkaHttpNoRules, null).scoped { eval => + val Right(result) = eval.apply(HelloWorldAkkaHttpNoRules.core.assembly) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + val allOccurrences = Seq( + referenceContent.contains("akka-http Reference Config File"), + referenceContent.contains("akka-http-core Reference Config File"), + referenceContent.contains("Akka Actor Reference Config File"), + referenceContent.contains("Akka Stream Reference Config File"), + referenceContent.contains("My application Reference Config File") + ) + + val timesOcccurres = allOccurrences.find(identity).size + + assert(timesOcccurres == 1) + } + } + + test("multiModule") - UnitTester( + HelloWorldMultiNoRules, + sourceRoot = helloWorldMultiResourcePath + ).scoped { eval => + val Right(result) = eval.apply(HelloWorldMultiNoRules.core.assembly) + + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + !referenceContent.contains("Model Reference Config File"), + !referenceContent.contains("foo.bar=2"), + referenceContent.contains("Core Reference Config File"), + referenceContent.contains("bar.baz=hello") + ) + } + } + } + } + +} diff --git a/scalalib/test/src/mill/scalalib/ScalaAssemblyTestUtils.scala b/scalalib/test/src/mill/scalalib/ScalaAssemblyTestUtils.scala new file mode 100644 index 00000000000..1de031bcd2f --- /dev/null +++ b/scalalib/test/src/mill/scalalib/ScalaAssemblyTestUtils.scala @@ -0,0 +1,107 @@ +package mill.scalalib + +import mill._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ + +import java.util.jar.JarFile +import scala.util.Using +import HelloWorldTests._ +trait ScalaAssemblyTestUtils { + + val akkaHttpDeps = Agg(ivy"com.typesafe.akka::akka-http:10.0.13") + + object HelloWorldAkkaHttpAppend extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) + } + } + + object HelloWorldAkkaHttpExclude extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) + } + } + + object HelloWorldAkkaHttpAppendPattern extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) + } + } + + object HelloWorldAkkaHttpExcludePattern extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) + } + } + + object HelloWorldAkkaHttpRelocate extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq(Assembly.Rule.Relocate("akka.**", "shaded.akka.@1")) + } + } + + object HelloWorldAkkaHttpNoRules extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def ivyDeps = akkaHttpDeps + override def assemblyRules = Seq.empty + } + } + + object HelloWorldMultiAppend extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiExclude extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiAppendPattern extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiAppendByPatternWithSeparator extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf", "\n")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiExcludePattern extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiNoRules extends TestBaseModule { + object core extends HelloWorldModuleWithMain { + override def moduleDeps = Seq(model) + override def assemblyRules = Seq.empty + } + object model extends HelloWorldModule + } + + val helloWorldMultiResourcePath = + os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-world-multi" + +} diff --git a/scalalib/test/src/mill/scalalib/ScalaAssemblyTests.scala b/scalalib/test/src/mill/scalalib/ScalaAssemblyTests.scala index 75a38958b20..6c6666eaae3 100644 --- a/scalalib/test/src/mill/scalalib/ScalaAssemblyTests.scala +++ b/scalalib/test/src/mill/scalalib/ScalaAssemblyTests.scala @@ -7,99 +7,7 @@ import utest._ import java.util.jar.JarFile import scala.util.Using import HelloWorldTests._ -object ScalaAssemblyTests extends TestSuite { - - val akkaHttpDeps = Agg(ivy"com.typesafe.akka::akka-http:10.0.13") - - object HelloWorldAkkaHttpAppend extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) - } - } - - object HelloWorldAkkaHttpExclude extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) - } - } - - object HelloWorldAkkaHttpAppendPattern extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) - } - } - - object HelloWorldAkkaHttpExcludePattern extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) - } - } - - object HelloWorldAkkaHttpRelocate extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq(Assembly.Rule.Relocate("akka.**", "shaded.akka.@1")) - } - } - - object HelloWorldAkkaHttpNoRules extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def ivyDeps = akkaHttpDeps - override def assemblyRules = Seq.empty - } - } - - object HelloWorldMultiAppend extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) - } - object model extends HelloWorldModule - } - - object HelloWorldMultiExclude extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) - } - object model extends HelloWorldModule - } - - object HelloWorldMultiAppendPattern extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) - } - object model extends HelloWorldModule - } - - object HelloWorldMultiAppendByPatternWithSeparator extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf", "\n")) - } - object model extends HelloWorldModule - } - - object HelloWorldMultiExcludePattern extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) - } - object model extends HelloWorldModule - } - - object HelloWorldMultiNoRules extends TestBaseModule { - object core extends HelloWorldModuleWithMain { - override def moduleDeps = Seq(model) - override def assemblyRules = Seq.empty - } - object model extends HelloWorldModule - } +object ScalaAssemblyTests extends TestSuite with ScalaAssemblyTestUtils { def tests: Tests = Tests { @@ -112,212 +20,16 @@ object ScalaAssemblyTests extends TestSuite { result.evalCount > 0 ) val jarFile = new JarFile(result.value.path.toIO) - val entries = jarEntries(jarFile) - - val mainPresent = entries.contains("Main.class") - assert(mainPresent) - assert(entries.exists(s => s.contains("scala/Predef.class"))) - - val mainClass = jarMainClass(jarFile) - assert(mainClass.contains("Main")) - } - - test("assemblyRules") { - def checkAppend[M <: mill.testkit.TestBaseModule](module: M, target: Target[PathRef]) = - UnitTester(module, resourcePath).scoped { eval => - val Right(result) = eval.apply(target) - - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(jarEntries(jarFile).contains("reference.conf")) - - val referenceContent = readFileFromJar(jarFile, "reference.conf") - - assert( - // akka modules configs are present - referenceContent.contains("akka-http Reference Config File"), - referenceContent.contains("akka-http-core Reference Config File"), - referenceContent.contains("Akka Actor Reference Config File"), - referenceContent.contains("Akka Stream Reference Config File"), - // our application config is present too - referenceContent.contains("My application Reference Config File"), - referenceContent.contains( - """akka.http.client.user-agent-header="hello-world-client"""" - ) - ) - } - } - - val helloWorldMultiResourcePath = - os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-world-multi" - - def checkAppendMulti[M <: mill.testkit.TestBaseModule]( - module: M, - target: Target[PathRef] - ): Unit = UnitTester( - module, - sourceRoot = helloWorldMultiResourcePath - ).scoped { eval => - val Right(result) = eval.apply(target) - - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(jarEntries(jarFile).contains("reference.conf")) - - val referenceContent = readFileFromJar(jarFile, "reference.conf") - - assert( - // reference config from core module - referenceContent.contains("Core Reference Config File"), - // reference config from model module - referenceContent.contains("Model Reference Config File"), - // concatenated content - referenceContent.contains("bar.baz=hello"), - referenceContent.contains("foo.bar=2") - ) - } - } - - def checkAppendWithSeparator[M <: mill.testkit.TestBaseModule]( - module: M, - target: Target[PathRef] - ): Unit = UnitTester( - module, - sourceRoot = helloWorldMultiResourcePath - ).scoped { eval => - val Right(result) = eval.apply(target) - - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(jarEntries(jarFile).contains("without-new-line.conf")) - - val result = readFileFromJar(jarFile, "without-new-line.conf").split('\n').toSet - val expected = Set("without-new-line.first=first", "without-new-line.second=second") - assert(result == expected) - } - } - - test("appendWithDeps") - checkAppend( - HelloWorldAkkaHttpAppend, - HelloWorldAkkaHttpAppend.core.assembly - ) - test("appendMultiModule") - checkAppendMulti( - HelloWorldMultiAppend, - HelloWorldMultiAppend.core.assembly - ) - test("appendPatternWithDeps") - checkAppend( - HelloWorldAkkaHttpAppendPattern, - HelloWorldAkkaHttpAppendPattern.core.assembly - ) - test("appendPatternMultiModule") - checkAppendMulti( - HelloWorldMultiAppendPattern, - HelloWorldMultiAppendPattern.core.assembly - ) - test("appendPatternMultiModuleWithSeparator") - checkAppendWithSeparator( - HelloWorldMultiAppendByPatternWithSeparator, - HelloWorldMultiAppendByPatternWithSeparator.core.assembly - ) - - def checkExclude[M <: mill.testkit.TestBaseModule]( - module: M, - target: Target[PathRef], - resourcePath: os.Path = resourcePath - ) = UnitTester(module, resourcePath).scoped { eval => - val Right(result) = eval.apply(target) + try { + val entries = jarEntries(jarFile) - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(!jarEntries(jarFile).contains("reference.conf")) - } - } + val mainPresent = entries.contains("Main.class") + assert(mainPresent) + assert(entries.exists(s => s.contains("scala/Predef.class"))) - test("excludeWithDeps") - checkExclude( - HelloWorldAkkaHttpExclude, - HelloWorldAkkaHttpExclude.core.assembly - ) - test("excludeMultiModule") - checkExclude( - HelloWorldMultiExclude, - HelloWorldMultiExclude.core.assembly, - resourcePath = helloWorldMultiResourcePath - ) - test("excludePatternWithDeps") - checkExclude( - HelloWorldAkkaHttpExcludePattern, - HelloWorldAkkaHttpExcludePattern.core.assembly - ) - test("excludePatternMultiModule") - checkExclude( - HelloWorldMultiExcludePattern, - HelloWorldMultiExcludePattern.core.assembly, - resourcePath = helloWorldMultiResourcePath - ) - - def checkRelocate[M <: mill.testkit.TestBaseModule]( - module: M, - target: Target[PathRef], - resourcePath: os.Path = resourcePath - ) = UnitTester(module, resourcePath).scoped { eval => - val Right(result) = eval.apply(target) - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(!jarEntries(jarFile).contains("akka/http/scaladsl/model/HttpEntity.class")) - assert( - jarEntries(jarFile).contains("shaded/akka/http/scaladsl/model/HttpEntity.class") - ) - } - } - - test("relocate") { - test("withDeps") - checkRelocate( - HelloWorldAkkaHttpRelocate, - HelloWorldAkkaHttpRelocate.core.assembly - ) - - test("run") - UnitTester( - HelloWorldAkkaHttpRelocate, - sourceRoot = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-world-deps" - ).scoped { eval => - val Right(result) = eval.apply(HelloWorldAkkaHttpRelocate.core.runMain("Main")) - assert(result.evalCount > 0) - } - } - - test("writeDownstreamWhenNoRule") { - test("withDeps") - UnitTester(HelloWorldAkkaHttpNoRules, null).scoped { eval => - val Right(result) = eval.apply(HelloWorldAkkaHttpNoRules.core.assembly) - - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(jarEntries(jarFile).contains("reference.conf")) - - val referenceContent = readFileFromJar(jarFile, "reference.conf") - - val allOccurrences = Seq( - referenceContent.contains("akka-http Reference Config File"), - referenceContent.contains("akka-http-core Reference Config File"), - referenceContent.contains("Akka Actor Reference Config File"), - referenceContent.contains("Akka Stream Reference Config File"), - referenceContent.contains("My application Reference Config File") - ) - - val timesOcccurres = allOccurrences.find(identity).size - - assert(timesOcccurres == 1) - } - } - - test("multiModule") - UnitTester( - HelloWorldMultiNoRules, - sourceRoot = helloWorldMultiResourcePath - ).scoped { eval => - val Right(result) = eval.apply(HelloWorldMultiNoRules.core.assembly) - - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - assert(jarEntries(jarFile).contains("reference.conf")) - - val referenceContent = readFileFromJar(jarFile, "reference.conf") - - assert( - !referenceContent.contains("Model Reference Config File"), - !referenceContent.contains("foo.bar=2"), - referenceContent.contains("Core Reference Config File"), - referenceContent.contains("bar.baz=hello") - ) - } - } - } + val mainClass = jarMainClass(jarFile) + assert(mainClass.contains("Main")) + } finally jarFile.close() } test("run") - UnitTester(HelloWorldTests.HelloWorldWithMain, resourcePath).scoped { eval => @@ -338,4 +50,5 @@ object ScalaAssemblyTests extends TestSuite { } } } + } diff --git a/scalalib/test/src/mill/scalalib/TestModuleUtilTests.scala b/scalalib/test/src/mill/scalalib/TestModuleUtilTests.scala index 5916910ae1a..730b4b7dac0 100644 --- a/scalalib/test/src/mill/scalalib/TestModuleUtilTests.scala +++ b/scalalib/test/src/mill/scalalib/TestModuleUtilTests.scala @@ -292,6 +292,110 @@ object TestModuleUtilTests extends TestSuite { ) ) } + + test("collapseTestClassNames") { + val res = TestModuleUtil.collapseTestClassNames( + Seq( + "mill.javalib.palantirformat.PalantirFormatModuleTest", + "mill.scalalib.AssemblyExeTests", + "mill.scalalib.AssemblyNoExeTests", + "mill.scalalib.CoursierMirrorTests", + "mill.scalalib.CrossVersionTests", + "mill.scalalib.CycleTests", + "mill.scalalib.DottyDocTests", + "mill.scalalib.HelloJavaTests", + "mill.scalalib.HelloWorldTests", + "mill.scalalib.PublishModuleTests", + "mill.scalalib.ResolveDepsTests", + "mill.scalalib.ScalaAmmoniteTests", + "mill.scalalib.ScalaAssemblyAppendTests", + "mill.scalalib.ScalaAssemblyExcludeTests", + "mill.scalalib.ScalaAssemblyTests", + "mill.scalalib.ScalaColorOutputTests", + "mill.scalalib.ScalaCrossVersionTests", + "mill.scalalib.ScalaDoc3Tests", + "mill.scalalib.ScalaDotty213Tests", + "mill.scalalib.ScalaFlagsTests", + "mill.scalalib.ScalaIvyDepsTests", + "mill.scalalib.ScalaMacrosTests", + "mill.scalalib.ScalaMultiModuleClasspathsTests", + "mill.scalalib.ScalaRunTests", + "mill.scalalib.ScalaScalacheckTests", + "mill.scalalib.ScalaScaladocTests", + "mill.scalalib.ScalaSemanticDbTests", + "mill.scalalib.ScalaTypeLevelTests", + "mill.scalalib.ScalaValidatedPathRefTests", + "mill.scalalib.ScalaVersionsRangesTests", + "mill.scalalib.ScalatestTestRunnerTests", + "mill.scalalib.TestClassLoaderTests", + "mill.scalalib.TestModuleUtilTests", + "mill.scalalib.UtestTestRunnerTests", + "mill.scalalib.VersionContolTests", + "mill.scalalib.ZiotestTestRunnerTests", + "mill.scalalib.api.ZincWorkerUtilTests", + "mill.scalalib.bsp.BspModuleTests", + "mill.scalalib.dependency.metadata.MetadataLoaderFactoryTests", + "mill.scalalib.dependency.updates.UpdatesFinderTests", + "mill.scalalib.dependency.versions.ScalaVersionTests", + "mill.scalalib.dependency.versions.VersionTests", + "mill.scalalib.giter8.Giter8Tests", + "mill.scalalib.publish.IvyTests", + "mill.scalalib.publish.LocalM2PublisherTests", + "mill.scalalib.publish.PomTests", + "mill.scalalib.scalafmt.ScalafmtTests" + ) + ) + val expected = Seq( + "mill.javalib.palantirformat.PalantirFormatModuleTest", + "m.scalalib.AssemblyExeTests", + "m.s.AssemblyNoExeTests", + "m.s.CoursierMirrorTests", + "m.s.CrossVersionTests", + "m.s.CycleTests", + "m.s.DottyDocTests", + "m.s.HelloJavaTests", + "m.s.HelloWorldTests", + "m.s.PublishModuleTests", + "m.s.ResolveDepsTests", + "m.s.ScalaAmmoniteTests", + "m.s.ScalaAssemblyAppendTests", + "m.s.ScalaAssemblyExcludeTests", + "m.s.ScalaAssemblyTests", + "m.s.ScalaColorOutputTests", + "m.s.ScalaCrossVersionTests", + "m.s.ScalaDoc3Tests", + "m.s.ScalaDotty213Tests", + "m.s.ScalaFlagsTests", + "m.s.ScalaIvyDepsTests", + "m.s.ScalaMacrosTests", + "m.s.ScalaMultiModuleClasspathsTests", + "m.s.ScalaRunTests", + "m.s.ScalaScalacheckTests", + "m.s.ScalaScaladocTests", + "m.s.ScalaSemanticDbTests", + "m.s.ScalaTypeLevelTests", + "m.s.ScalaValidatedPathRefTests", + "m.s.ScalaVersionsRangesTests", + "m.s.ScalatestTestRunnerTests", + "m.s.TestClassLoaderTests", + "m.s.TestModuleUtilTests", + "m.s.UtestTestRunnerTests", + "m.s.VersionContolTests", + "m.s.ZiotestTestRunnerTests", + "m.s.api.ZincWorkerUtilTests", + "m.s.bsp.BspModuleTests", + "m.s.dependency.metadata.MetadataLoaderFactoryTests", + "m.s.d.updates.UpdatesFinderTests", + "m.s.d.versions.ScalaVersionTests", + "m.s.d.v.VersionTests", + "m.s.giter8.Giter8Tests", + "m.s.publish.IvyTests", + "m.s.p.LocalM2PublisherTests", + "m.s.p.PomTests", + "m.s.scalafmt.ScalafmtTests" + ) + assert(expected == res) + } } private def assertEquals(expected: Elem, actual: Option[Elem]): Unit = { diff --git a/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala new file mode 100644 index 00000000000..7736b918cd6 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala @@ -0,0 +1,59 @@ +package mill.scalalib + +import mill.api.Result +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import mill.{Agg, T, Task} +import os.Path +import sbt.testing.Status +import utest._ + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.xml.{Elem, NodeSeq, XML} + +object TestRunnerScalatestTests extends TestSuite { + import TestRunnerTestUtils._ + override def tests: Tests = Tests { + test("ScalaTest") { + test("test") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval(testrunner.scalatest.test()) + assert(result.value._2.size == 3) + junitReportIn(eval.outPath, "scalatest").shouldHave(3 -> Status.Success) + } + test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval.apply(testrunner.scalatest.discoveredTestClasses) + val expected = Seq("mill.scalalib.ScalaTestSpec") + assert(result.value == expected) + expected + } + + test("testOnly") - { + val tester = new TestOnlyTester(_.scalatest) + + test("all") - tester.testOnly(Seq("mill.scalalib.ScalaTestSpec"), 3) + test("include") - tester.testOnly( + Seq("mill.scalalib.ScalaTestSpec", "--", "-n", "tagged"), + 1 + ) + test("exclude") - tester.testOnly( + Seq("mill.scalalib.ScalaTestSpec", "--", "-l", "tagged"), + 2 + ) + test("includeAndExclude") - tester.testOnly0 { (eval, mod) => + val Left(Result.Failure(msg, _)) = + eval.apply(mod.scalatest.testOnly( + "mill.scalalib.ScalaTestSpec", + "--", + "-n", + "tagged", + "-l", + "tagged" + )) + assert(msg.contains("Test selector does not match any test")) + } + } + } + + } + +} diff --git a/scalalib/test/src/mill/scalalib/TestRunnerTestUtils.scala b/scalalib/test/src/mill/scalalib/TestRunnerTestUtils.scala new file mode 100644 index 00000000000..9e8fdc8ea42 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/TestRunnerTestUtils.scala @@ -0,0 +1,136 @@ +package mill.scalalib + +import mill.api.Result +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import mill.{Agg, T, Task} +import os.Path +import sbt.testing.Status +import utest._ + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.xml.{Elem, NodeSeq, XML} + +object TestRunnerTestUtils { + object testrunner extends TestRunnerTestModule { + def computeTestForkGrouping(x: Seq[String]) = Seq(x) + } + + object testrunnerGrouping extends TestRunnerTestModule { + def computeTestForkGrouping(x: Seq[String]) = x.sorted.grouped(2).toSeq + } + + trait TestRunnerTestModule extends TestBaseModule with ScalaModule { + def computeTestForkGrouping(x: Seq[String]): Seq[Seq[String]] + def scalaVersion = sys.props.getOrElse("TEST_SCALA_2_13_VERSION", ???) + + object utest extends ScalaTests with TestModule.Utest { + override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) + override def ivyDeps = Task { + super.ivyDeps() ++ Agg( + ivy"com.lihaoyi::utest:${sys.props.getOrElse("TEST_UTEST_VERSION", ???)}" + ) + } + } + + object scalatest extends ScalaTests with TestModule.ScalaTest { + override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) + override def ivyDeps = Task { + super.ivyDeps() ++ Agg( + ivy"org.scalatest::scalatest:${sys.props.getOrElse("TEST_SCALATEST_VERSION", ???)}" + ) + } + } + + trait DoneMessage extends ScalaTests { + override def ivyDeps = Task { + super.ivyDeps() ++ Agg( + ivy"org.scala-sbt:test-interface:${sys.props.getOrElse("TEST_TEST_INTERFACE_VERSION", ???)}" + ) + } + } + object doneMessageSuccess extends DoneMessage { + def testFramework = "mill.scalalib.DoneMessageSuccessFramework" + } + object doneMessageFailure extends DoneMessage { + def testFramework = "mill.scalalib.DoneMessageFailureFramework" + } + object doneMessageNull extends DoneMessage { + def testFramework = "mill.scalalib.DoneMessageNullFramework" + } + + object ziotest extends ScalaTests with TestModule.ZioTest { + override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) + override def ivyDeps = Task { + super.ivyDeps() ++ Agg( + ivy"dev.zio::zio-test:${sys.props.getOrElse("TEST_ZIOTEST_VERSION", ???)}", + ivy"dev.zio::zio-test-sbt:${sys.props.getOrElse("TEST_ZIOTEST_VERSION", ???)}" + ) + } + } + } + + val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "testrunner" + + class TestOnlyTester(m: TestRunnerTestModule => TestModule) { + def testOnly0(f: (UnitTester, TestRunnerTestModule) => Unit) = { + for (mod <- Seq(testrunner, testrunnerGrouping)) { + UnitTester(mod, resourcePath).scoped { eval => f(eval, mod) } + } + } + def testOnly( + args: Seq[String], + size: Int, + expectedFileListing: Map[TestModule, Set[String]] = Map() + ) = { + testOnly0 { (eval, mod) => + val Right(result) = eval.apply(m(mod).testOnly(args: _*)) + val testOnly = result.value + if (expectedFileListing.nonEmpty) { + val dest = eval.outPath / m(mod).toString / "testOnly.dest" + val sortedListed = os.list(dest).map(_.last).sorted + val sortedExpected = expectedFileListing(m(mod)).toSeq.sorted + assert(sortedListed == sortedExpected) + } + // Regardless of whether tests are grouped or not, the same + // number of test results appear at the end + assert(testOnly._2.size == size) + } + } + } + trait JUnitReportMatch { + def shouldHave(statuses: (Int, Status)*): Unit + } + def junitReportIn( + outPath: Path, + moduleName: String, + action: String = "test" + ): JUnitReportMatch = { + val reportPath: Path = outPath / moduleName / s"$action.dest" / "test-report.xml" + val reportXML = XML.loadFile(reportPath.toIO) + new JUnitReportMatch { + override def shouldHave(statuses: (Int, Status)*): Unit = { + def getValue(attribute: String): Int = + reportXML.attribute(attribute).map(_.toString).getOrElse("0").toInt + statuses.foreach { case (expectedQuantity: Int, status: Status) => + status match { + case Status.Success => + val testCases: NodeSeq = reportXML \\ "testcase" + val actualSucceededTestCases: Int = + testCases.count(tc => !tc.child.exists(n => n.isInstanceOf[Elem])) + assert(expectedQuantity == actualSucceededTestCases) + case _ => + val statusXML = reportXML \\ status.name().toLowerCase + val nbSpecificStatusElement = statusXML.size + assert(expectedQuantity == nbSpecificStatusElement) + val specificStatusAttributeValue = getValue(s"${status.name().toLowerCase}s") + assert(expectedQuantity == specificStatusAttributeValue) + } + } + val expectedNbTests = statuses.map(_._1).sum + val actualNbTests = getValue("tests") + assert(expectedNbTests == actualNbTests) + } + } + } +} diff --git a/scalalib/test/src/mill/scalalib/TestRunnerTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerTests.scala deleted file mode 100644 index c85fa702b1c..00000000000 --- a/scalalib/test/src/mill/scalalib/TestRunnerTests.scala +++ /dev/null @@ -1,296 +0,0 @@ -package mill.scalalib - -import mill.api.Result -import mill.testkit.UnitTester -import mill.testkit.TestBaseModule -import mill.{Agg, T, Task} -import os.Path -import sbt.testing.Status -import utest._ - -import java.io.{ByteArrayOutputStream, PrintStream} -import scala.xml.{Elem, NodeSeq, XML} - -object TestRunnerTests extends TestSuite { - object testrunner extends TestRunnerTestModule { - def computeTestForkGrouping(x: Seq[String]) = Seq(x) - } - - object testrunnerGrouping extends TestRunnerTestModule { - def computeTestForkGrouping(x: Seq[String]) = x.sorted.grouped(2).toSeq - } - - trait TestRunnerTestModule extends TestBaseModule with ScalaModule { - def computeTestForkGrouping(x: Seq[String]): Seq[Seq[String]] - def scalaVersion = sys.props.getOrElse("TEST_SCALA_2_13_VERSION", ???) - - object utest extends ScalaTests with TestModule.Utest { - override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) - override def ivyDeps = Task { - super.ivyDeps() ++ Agg( - ivy"com.lihaoyi::utest:${sys.props.getOrElse("TEST_UTEST_VERSION", ???)}" - ) - } - } - - object scalatest extends ScalaTests with TestModule.ScalaTest { - override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) - override def ivyDeps = Task { - super.ivyDeps() ++ Agg( - ivy"org.scalatest::scalatest:${sys.props.getOrElse("TEST_SCALATEST_VERSION", ???)}" - ) - } - } - - trait DoneMessage extends ScalaTests { - override def ivyDeps = Task { - super.ivyDeps() ++ Agg( - ivy"org.scala-sbt:test-interface:${sys.props.getOrElse("TEST_TEST_INTERFACE_VERSION", ???)}" - ) - } - } - object doneMessageSuccess extends DoneMessage { - def testFramework = "mill.scalalib.DoneMessageSuccessFramework" - } - object doneMessageFailure extends DoneMessage { - def testFramework = "mill.scalalib.DoneMessageFailureFramework" - } - object doneMessageNull extends DoneMessage { - def testFramework = "mill.scalalib.DoneMessageNullFramework" - } - - object ziotest extends ScalaTests with TestModule.ZioTest { - override def testForkGrouping = computeTestForkGrouping(discoveredTestClasses()) - override def ivyDeps = Task { - super.ivyDeps() ++ Agg( - ivy"dev.zio::zio-test:${sys.props.getOrElse("TEST_ZIOTEST_VERSION", ???)}", - ivy"dev.zio::zio-test-sbt:${sys.props.getOrElse("TEST_ZIOTEST_VERSION", ???)}" - ) - } - } - } - - val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "testrunner" - - override def tests: Tests = Tests { - test("TestRunner") - { - test("utest") - { - test("test case lookup") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval.apply(testrunner.utest.test()) - val test = result.value.asInstanceOf[(String, Seq[mill.testrunner.TestResult])] - assert( - test._2.size == 3 - ) - junitReportIn(eval.outPath, "utest").shouldHave(3 -> Status.Success) - } - test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval.apply(testrunner.utest.discoveredTestClasses) - val expected = Seq( - "mill.scalalib.BarTests", - "mill.scalalib.FooTests", - "mill.scalalib.FoobarTests" - ) - assert(result.value == expected) - expected - } - test("testOnly") - { - val tester = new TestOnlyTester(_.utest) - test("suffix") - tester.testOnly(Seq("*arTests"), 2) - test("prefix") - tester.testOnly(Seq("mill.scalalib.FooT*"), 1) - test("exactly") - tester.testOnly( - Seq("mill.scalalib.FooTests"), - 1, - Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), - // When there is only one test group with test classes, we do not put it in a subfolder - testrunnerGrouping.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs") - ) - ) - test("multi") - tester.testOnly( - Seq("*Bar*", "*bar*"), - 2, - Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), - // When there are multiple test groups with one test class each, we - // put each test group in a subfolder with the number of the class - testrunnerGrouping.utest -> Set( - "mill.scalalib.BarTests", - "mill.scalalib.FoobarTests", - "test-report.xml" - ) - ) - ) - test("all") - tester.testOnly( - Seq("*"), - 3, - Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), - // When there are multiple test groups some with multiple test classes, we put each - // test group in a subfolder with the index of the group, and for any test groups - // with only one test class we append the name of the class - testrunnerGrouping.utest -> Set( - "group-0", - "group-1-mill.scalalib.FoobarTests", - "test-report.xml" - ) - ) - ) - test("noMatch") - tester.testOnly0 { (eval, mod) => - val Left(Result.Failure(msg, _)) = - eval.apply(mod.utest.testOnly("noMatch", "noMatch*2")) - assert( - msg == "Test selector does not match any test: noMatch noMatch*2\nRun discoveredTestClasses to see available tests" - ) - } - } - } - - test("doneMessage") { - test("failure") { - val outStream = new ByteArrayOutputStream() - UnitTester( - testrunner, - outStream = new PrintStream(outStream, true), - sourceRoot = resourcePath - ).scoped { eval => - val Left(Result.Failure(msg, _)) = eval(testrunner.doneMessageFailure.test()) - val stdout = new String(outStream.toByteArray) - assert(stdout.contains("test failure done message")) - junitReportIn(eval.outPath, "doneMessageFailure").shouldHave(1 -> Status.Failure) - } - } - test("success") { - val outStream = new ByteArrayOutputStream() - UnitTester( - testrunner, - outStream = new PrintStream(outStream, true), - sourceRoot = resourcePath - ).scoped { eval => - val Right(_) = eval(testrunner.doneMessageSuccess.test()) - val stdout = new String(outStream.toByteArray) - assert(stdout.contains("test success done message")) - } - } - - test("null") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(_) = eval(testrunner.doneMessageNull.test()) - } - } - test("ScalaTest") { - test("test") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval(testrunner.scalatest.test()) - assert(result.value._2.size == 3) - junitReportIn(eval.outPath, "scalatest").shouldHave(3 -> Status.Success) - } - test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval.apply(testrunner.scalatest.discoveredTestClasses) - val expected = Seq("mill.scalalib.ScalaTestSpec") - assert(result.value == expected) - expected - } - - test("testOnly") - { - val tester = new TestOnlyTester(_.scalatest) - - test("all") - tester.testOnly(Seq("mill.scalalib.ScalaTestSpec"), 3) - test("include") - tester.testOnly( - Seq("mill.scalalib.ScalaTestSpec", "--", "-n", "tagged"), - 1 - ) - test("exclude") - tester.testOnly( - Seq("mill.scalalib.ScalaTestSpec", "--", "-l", "tagged"), - 2 - ) - test("includeAndExclude") - tester.testOnly0 { (eval, mod) => - val Left(Result.Failure(msg, _)) = - eval.apply(mod.scalatest.testOnly( - "mill.scalalib.ScalaTestSpec", - "--", - "-n", - "tagged", - "-l", - "tagged" - )) - assert(msg.contains("Test selector does not match any test")) - } - } - } - - test("ZioTest") { - test("test") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval(testrunner.ziotest.test()) - assert(result.value._2.size == 1) - junitReportIn(eval.outPath, "ziotest").shouldHave(1 -> Status.Success) - } - test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => - val Right(result) = eval.apply(testrunner.ziotest.discoveredTestClasses) - val expected = Seq("mill.scalalib.ZioTestSpec") - assert(result.value == expected) - expected - } - } - } - } - - class TestOnlyTester(m: TestRunnerTestModule => TestModule) { - def testOnly0(f: (UnitTester, TestRunnerTestModule) => Unit) = { - for (mod <- Seq(testrunner, testrunnerGrouping)) { - UnitTester(mod, resourcePath).scoped { eval => f(eval, mod) } - } - } - def testOnly( - args: Seq[String], - size: Int, - expectedFileListing: Map[TestModule, Set[String]] = Map() - ) = { - testOnly0 { (eval, mod) => - val Right(result) = eval.apply(m(mod).testOnly(args: _*)) - val testOnly = result.value - if (expectedFileListing.nonEmpty) { - val dest = eval.outPath / m(mod).toString / "testOnly.dest" - val sortedListed = os.list(dest).map(_.last).sorted - val sortedExpected = expectedFileListing(m(mod)).toSeq.sorted - assert(sortedListed == sortedExpected) - } - // Regardless of whether tests are grouped or not, the same - // number of test results appear at the end - assert(testOnly._2.size == size) - } - } - } - trait JUnitReportMatch { - def shouldHave(statuses: (Int, Status)*): Unit - } - private def junitReportIn( - outPath: Path, - moduleName: String, - action: String = "test" - ): JUnitReportMatch = { - val reportPath: Path = outPath / moduleName / s"$action.dest" / "test-report.xml" - val reportXML = XML.loadFile(reportPath.toIO) - new JUnitReportMatch { - override def shouldHave(statuses: (Int, Status)*): Unit = { - def getValue(attribute: String): Int = - reportXML.attribute(attribute).map(_.toString).getOrElse("0").toInt - statuses.foreach { case (expectedQuantity: Int, status: Status) => - status match { - case Status.Success => - val testCases: NodeSeq = reportXML \\ "testcase" - val actualSucceededTestCases: Int = - testCases.count(tc => !tc.child.exists(n => n.isInstanceOf[Elem])) - assert(expectedQuantity == actualSucceededTestCases) - case _ => - val statusXML = reportXML \\ status.name().toLowerCase - val nbSpecificStatusElement = statusXML.size - assert(expectedQuantity == nbSpecificStatusElement) - val specificStatusAttributeValue = getValue(s"${status.name().toLowerCase}s") - assert(expectedQuantity == specificStatusAttributeValue) - } - } - val expectedNbTests = statuses.map(_._1).sum - val actualNbTests = getValue("tests") - assert(expectedNbTests == actualNbTests) - } - } - } -} diff --git a/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala new file mode 100644 index 00000000000..b495a96cc7f --- /dev/null +++ b/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala @@ -0,0 +1,121 @@ +package mill.scalalib + +import mill.api.Result +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import mill.{Agg, T, Task} +import os.Path +import sbt.testing.Status +import utest._ + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.xml.{Elem, NodeSeq, XML} + +object TestRunnerUtestTests extends TestSuite { + import TestRunnerTestUtils._ + override def tests: Tests = Tests { + test("utest") - { + test("test case lookup") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval.apply(testrunner.utest.test()) + val test = result.value.asInstanceOf[(String, Seq[mill.testrunner.TestResult])] + assert( + test._2.size == 3 + ) + junitReportIn(eval.outPath, "utest").shouldHave(3 -> Status.Success) + } + test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval.apply(testrunner.utest.discoveredTestClasses) + val expected = Seq( + "mill.scalalib.BarTests", + "mill.scalalib.FooTests", + "mill.scalalib.FoobarTests" + ) + assert(result.value == expected) + expected + } + test("testOnly") - { + val tester = new TestOnlyTester(_.utest) + test("suffix") - tester.testOnly(Seq("*arTests"), 2) + test("prefix") - tester.testOnly(Seq("mill.scalalib.FooT*"), 1) + test("exactly") - tester.testOnly( + Seq("mill.scalalib.FooTests"), + 1, + Map( + testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + // When there is only one test group with test classes, we do not put it in a subfolder + testrunnerGrouping.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs") + ) + ) + test("multi") - tester.testOnly( + Seq("*Bar*", "*bar*"), + 2, + Map( + testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + // When there are multiple test groups with one test class each, we + // put each test group in a subfolder with the number of the class + testrunnerGrouping.utest -> Set( + "mill.scalalib.BarTests", + "mill.scalalib.FoobarTests", + "test-report.xml" + ) + ) + ) + test("all") - tester.testOnly( + Seq("*"), + 3, + Map( + testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + // When there are multiple test groups some with multiple test classes, we put each + // test group in a subfolder with the index of the group, and for any test groups + // with only one test class we append the name of the class + testrunnerGrouping.utest -> Set( + "group-0-mill.scalalib.BarTests", + "mill.scalalib.FoobarTests", + "test-report.xml" + ) + ) + ) + test("noMatch") - tester.testOnly0 { (eval, mod) => + val Left(Result.Failure(msg, _)) = + eval.apply(mod.utest.testOnly("noMatch", "noMatch*2")) + assert( + msg == "Test selector does not match any test: noMatch noMatch*2\nRun discoveredTestClasses to see available tests" + ) + } + } + } + + test("doneMessage") { + test("failure") { + val outStream = new ByteArrayOutputStream() + UnitTester( + testrunner, + outStream = new PrintStream(outStream, true), + sourceRoot = resourcePath + ).scoped { eval => + val Left(Result.Failure(msg, _)) = eval(testrunner.doneMessageFailure.test()) + val stdout = new String(outStream.toByteArray) + assert(stdout.contains("test failure done message")) + junitReportIn(eval.outPath, "doneMessageFailure").shouldHave(1 -> Status.Failure) + } + } + test("success") { + val outStream = new ByteArrayOutputStream() + UnitTester( + testrunner, + outStream = new PrintStream(outStream, true), + sourceRoot = resourcePath + ).scoped { eval => + val Right(_) = eval(testrunner.doneMessageSuccess.test()) + val stdout = new String(outStream.toByteArray) + assert(stdout.contains("test success done message")) + } + } + + test("null") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(_) = eval(testrunner.doneMessageNull.test()) + } + } + } + +} diff --git a/scalalib/test/src/mill/scalalib/TestRunnerZiotestTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerZiotestTests.scala new file mode 100644 index 00000000000..d1f129bb33a --- /dev/null +++ b/scalalib/test/src/mill/scalalib/TestRunnerZiotestTests.scala @@ -0,0 +1,32 @@ +package mill.scalalib + +import mill.api.Result +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import mill.{Agg, T, Task} +import os.Path +import sbt.testing.Status +import utest._ + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.xml.{Elem, NodeSeq, XML} + +object TestRunnerZiotestTests extends TestSuite { + import TestRunnerTestUtils._ + override def tests: Tests = Tests { + test("ZioTest") { + test("test") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval(testrunner.ziotest.test()) + assert(result.value._2.size == 1) + junitReportIn(eval.outPath, "ziotest").shouldHave(1 -> Status.Success) + } + test("discoveredTestClasses") - UnitTester(testrunner, resourcePath).scoped { eval => + val Right(result) = eval.apply(testrunner.ziotest.discoveredTestClasses) + val expected = Seq("mill.scalalib.ZioTestSpec") + assert(result.value == expected) + expected + } + } + } + +}