diff --git a/build.sbt b/build.sbt index 814e197d7fc7..f264f7f48804 100644 --- a/build.sbt +++ b/build.sbt @@ -3691,9 +3691,19 @@ lazy val `engine-runner` = project val epbLang = (`runtime-language-epb` / Compile / fullClasspath).value .map(_.data.getAbsolutePath) - val langServer = - (`language-server` / Compile / fullClasspath).value + def langServer = { + val log = streams.value.log + val path = (`language-server` / Compile / fullClasspath).value .map(_.data.getAbsolutePath) + if (GraalVM.EnsoLauncher.disableLanguageServer) { + log.info( + s"Skipping language server in native image build as ${GraalVM.EnsoLauncher.VAR_NAME} env variable is ${GraalVM.EnsoLauncher.toString}" + ) + Seq() + } else { + path + } + } val core = ( runnerDeps ++ runtimeDeps ++ @@ -3822,7 +3832,7 @@ lazy val `engine-runner` = project .dependsOn(NativeImage.additionalCp) .dependsOn(NativeImage.smallJdk) .dependsOn( - createEnginePackage + createEnginePackageNoIndex ) .value, buildNativeImage := Def.taskDyn { @@ -5106,6 +5116,7 @@ lazy val createEnginePackage = taskKey[Unit]("Creates the engine distribution package") createEnginePackage := { updateLibraryManifests.value + buildEngineDistributionNoIndex.value val modulesToCopy = componentModulesPaths.value val root = engineDistributionRoot.value val log = streams.value.log @@ -5131,48 +5142,49 @@ ThisBuild / createEnginePackage := { createEnginePackage.result.value } -lazy val buildEngineDistribution = - taskKey[Unit]("Builds the engine distribution and optionally native image") -buildEngineDistribution := Def.taskIf { +lazy val createEnginePackageNoIndex = + taskKey[Unit]("Creates the engine distribution package") +createEnginePackageNoIndex := { + updateLibraryManifests.value + val modulesToCopy = componentModulesPaths.value + val root = engineDistributionRoot.value + val log = streams.value.log + val cacheFactory = streams.value.cacheStoreFactory + DistributionPackage.createEnginePackage( + distributionRoot = root, + cacheFactory = cacheFactory, + log = log, + jarModulesToCopy = modulesToCopy, + graalVersion = graalMavenPackagesVersion, + javaVersion = graalVersion, + ensoVersion = ensoVersion, + editionName = currentEdition, + sourceStdlibVersion = stdLibVersion, + targetStdlibVersion = targetStdlibVersion, + targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value, + generateIndex = false + ) + log.info(s"Engine package created at $root") +} + +ThisBuild / createEnginePackageNoIndex := { + createEnginePackageNoIndex.result.value +} + +lazy val buildEngineDistributionNoIndex = + taskKey[Unit]( + "Builds the engine distribution without generating indexes and optionally generating native image" + ) +buildEngineDistributionNoIndex := Def.taskIf { + createEnginePackageNoIndex.value if (shouldBuildNativeImage.value) { - createEnginePackage.value (`engine-runner` / buildNativeImage).value - } else { - createEnginePackage.value } }.value // This makes the buildEngineDistribution task usable as a dependency // of other tasks. -ThisBuild / buildEngineDistribution := { - buildEngineDistribution.result.value -} - -lazy val shouldBuildNativeImage = taskKey[Boolean]( - "Whether native image should be build within buildEngineDistribution task" -) - -ThisBuild / shouldBuildNativeImage := { - val prop = System.getenv("ENSO_LAUNCHER") - prop == "native" || prop == "debugnative" -} - -ThisBuild / NativeImage.additionalOpts := { - val prop = System.getenv("ENSO_LAUNCHER") - if (prop == "native") { - Seq("-O3") - } else { - Seq("-ea", "-Ob", "-H:GenerateDebugInfo=1") - } -} - -ThisBuild / engineDistributionRoot := { - engineDistributionRoot.value -} - -lazy val buildEngineDistributionNoIndex = - taskKey[Unit]("Builds the engine distribution without generating indexes") -buildEngineDistributionNoIndex := { +ThisBuild / buildEngineDistributionNoIndex := { updateLibraryManifests.value val modulesToCopy = componentModulesPaths.value val root = engineDistributionRoot.value @@ -5195,10 +5207,49 @@ buildEngineDistributionNoIndex := { log.info(s"Engine package created at $root") } +lazy val shouldBuildNativeImage = taskKey[Boolean]( + "Whether native image should be build within buildEngineDistribution task" +) + +ThisBuild / shouldBuildNativeImage := { + GraalVM.EnsoLauncher.native +} + +ThisBuild / NativeImage.additionalOpts := { + if (GraalVM.EnsoLauncher.shell) { + Seq() + } else { + var opts = if (GraalVM.EnsoLauncher.release) { + Seq("-O3") + } else { + Seq("-Ob") + } + + if (GraalVM.EnsoLauncher.debug) { + opts = opts ++ Seq("-H:GenerateDebugInfo=1") + } + if (GraalVM.EnsoLauncher.test) { + opts = opts ++ Seq("-ea") + } + opts + } +} + +ThisBuild / engineDistributionRoot := { + engineDistributionRoot.value +} + +lazy val buildEngineDistribution = + taskKey[Unit]("Builds the engine distribution") +buildEngineDistribution := { + buildEngineDistributionNoIndex.value + createEnginePackage.value +} + // This makes the buildEngineDistributionNoIndex task usable as a dependency // of other tasks. -ThisBuild / buildEngineDistributionNoIndex := { - buildEngineDistributionNoIndex.result.value +ThisBuild / buildEngineDistribution := { + buildEngineDistribution.result.value } lazy val runEngineDistribution = @@ -5284,6 +5335,7 @@ buildStdLib := Def.inputTaskDyn { lazy val pkgStdLibInternal = inputKey[Unit]("Use `buildStdLib`") pkgStdLibInternal := Def.inputTask { + buildEngineDistributionNoIndex.value val cmd = allStdBits.parsed val root = engineDistributionRoot.value val log: sbt.Logger = streams.value.log diff --git a/build_tools/build/src/engine.rs b/build_tools/build/src/engine.rs index d55028be10fb..3e812ab42ce3 100644 --- a/build_tools/build/src/engine.rs +++ b/build_tools/build/src/engine.rs @@ -180,43 +180,33 @@ pub enum EngineLauncher { /// The binary inside the engine distribution will be built as an optimized native image Native, /// The binary inside the engine distribution will be built as native image with assertions + /// enabled but no debug information + TestNative, + /// The binary inside the engine distribution will be built as native image with assertions /// enabled and debug information - DebugNative, + TestDebugNative, /// The binary inside the engine distribution will be a shell script #[default] Shell, } - impl FromStr for EngineLauncher { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - match s { - "native" => Ok(Self::Native), - "debugnative" => Ok(Self::DebugNative), - "shell" => Ok(Self::Shell), - _ => bail!("Invalid Engine Launcher type: {}", s), - } + bail!("Parsing of ENSO_LAUNCHER isn't needed: {}", s) } } -impl From for String { - fn from(value: EngineLauncher) -> Self { - match value { +impl Display for EngineLauncher { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + let str = match self { EngineLauncher::Native => "native".to_string(), - EngineLauncher::DebugNative => "debugnative".to_string(), + EngineLauncher::TestNative => "native,test".to_string(), + EngineLauncher::TestDebugNative => "native,test,debug".to_string(), EngineLauncher::Shell => "shell".to_string(), - } - } -} + }; -impl Display for EngineLauncher { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - match *self { - EngineLauncher::Native => write!(f, "native"), - EngineLauncher::DebugNative => write!(f, "debugnative"), - EngineLauncher::Shell => write!(f, "shell"), - } + write!(f, "{}", str) } } diff --git a/build_tools/build/src/engine/context.rs b/build_tools/build/src/engine/context.rs index ee8d6b4581ea..3e15cc6c28c4 100644 --- a/build_tools/build/src/engine/context.rs +++ b/build_tools/build/src/engine/context.rs @@ -200,7 +200,7 @@ impl RunContext { sbt.call_arg("syntax-rust-definition/Runtime/managedClasspath").await?; } if self.config.build_native_runner { - env::ENSO_LAUNCHER.set(&engine::EngineLauncher::DebugNative)?; + env::ENSO_LAUNCHER.set(&engine::EngineLauncher::TestNative)?; } // TODO: Once the native image is production ready, we should switch to diff --git a/docs/infrastructure/native-image.md b/docs/infrastructure/native-image.md index aa9d5be6fd18..2605cc34169a 100644 --- a/docs/infrastructure/native-image.md +++ b/docs/infrastructure/native-image.md @@ -206,25 +206,30 @@ one of the following: - `shell`: The default value. `buildEngineDistribution` command does not build the native image. -- `debugnative`: `buildEngineDistribution` command builds native image with - assertions enabled (`-ea`). Useful for running tests on the CI. -- `native`: `buildEngineDistribution` command builds native image with - assertions disabled (`-ea`). Turns on maximal optimizations which may increase - the build time. - -To generate the Native Image for runner either explicitly execute - -```bash -sbt> engine-runner/buildNativeImage -``` - -or +- `native`: `buildEngineDistribution` command builds native image in _release + mode_ - e.g. turns on maximal optimizations increasing the build time. +- There are additional variants of `native` useful for _development_. They are + specified as comma separated attributes following `native`: + - using `native,fast` turns on _native image_ build, but disables + optimizations - e.g. produces build similar to _release mode_, but more + quickly + - using `native,test` _enables assertions_ - e.g. it instructs + `buildEngineDistribution` command to build native image with assertions + enabled (`-ea`). Useful for running Enso tests in the _native mode_. + - using `native,debug` generates _debugging informations_ for VSCode _native + image debugger_ + - using `native,-ls` disables support for _language server_ in the generated + binary + - it is possible to combine all features - e.g. use `debug,fast,test,native` + +To test _native image_ launcher choose one of the `native` configurations and +invoke: ```bash $ ENSO_LAUNCHER=native sbt buildEngineDistribution ``` -and execute any program with that binary - for example `test/Base_Tests` +then execute any program with that binary - for example `test/Base_Tests` ```bash $ ./built-distribution/enso-engine-*/enso-*/bin/enso --run test/Base_Tests diff --git a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java index f9fa7eb67875..49a9c87b0e92 100644 --- a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java +++ b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java @@ -5,6 +5,7 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.LinkedHashSet; import java.util.TreeSet; import org.enso.compiler.core.EnsoParser; @@ -110,7 +111,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs()); for (var nativeLib : nativeLibs) { var out = new File(nativeLibDir, nativeLib.getName()); - Files.copy(nativeLib.toPath(), out.toPath()); + Files.copy(nativeLib.toPath(), out.toPath(), StandardCopyOption.REPLACE_EXISTING); nativeLibPaths.add(out.getAbsolutePath()); } } diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index fe827168f626..5af65ec471e3 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -95,11 +95,19 @@ object DistributionPackage { } } - def executableName(baseName: String): String = + private def executableName(baseName: String): String = if (Platform.isWindows) baseName + ".exe" else baseName - private def batName(baseName: String): String = - if (Platform.isWindows) baseName + ".bat" else baseName + private def batOrExeName(baseName: String): String = + if (Platform.isWindows) { + if (GraalVM.EnsoLauncher.native) { + baseName + ".exe" + } else { + baseName + ".bat" + } + } else { + baseName + } def createProjectManagerPackage( distributionRoot: File, @@ -169,11 +177,17 @@ object DistributionPackage { log = log ) - copyDirectoryIncremental( - file("distribution/bin"), - distributionRoot / "bin", - cacheFactory.make("engine-bin") - ) + if (!GraalVM.EnsoLauncher.shell) { + log.info( + s"Not using shell launchers as ${GraalVM.EnsoLauncher.VAR_NAME} env variable is ${GraalVM.EnsoLauncher.toString}" + ) + } else { + copyDirectoryIncremental( + file("distribution/bin"), + distributionRoot / "bin", + cacheFactory.make("engine-bin") + ) + } buildEngineManifest( template = file("distribution/manifest.template.yaml"), @@ -235,59 +249,89 @@ object DistributionPackage { ) { diff => if (diff.modified.nonEmpty) { log.info(s"Generating index for $libName ") + val fileToExecute = new File( + ensoExecutable.getParentFile, + batOrExeName(ensoExecutable.getName) + ) + + def assertExecutable(when: String) = { + if (!fileToExecute.canExecute()) { + log.warn(s"Not an executable file ${fileToExecute} $when") + var dir = fileToExecute + while (dir != null && !dir.exists()) { + dir = dir.getParentFile + } + var count = 0 + if (dir != null) { + log.warn(s"Content of ${dir}") + Option(dir.listFiles).map(_.map { file => + log.warn(s" ${file}") + count += 1 + }) + } + log.warn(s"Found ${count} files.") + } + } + assertExecutable("before launching") val command = Seq( - Platform.executableFile(ensoExecutable.getAbsoluteFile), + fileToExecute.getAbsolutePath, "--no-compile-dependencies", "--no-global-cache", "--compile", path.getAbsolutePath ) log.debug(command.mkString(" ")) - val runningProcess = Process( - command, - Some(path.getAbsoluteFile.getParentFile), - "JAVA_OPTS" -> "-Dorg.jline.terminal.dumb=true" - ).run - // Poor man's solution to stuck index generation - val GENERATING_INDEX_TIMEOUT = 60 * 2 // 2 minutes - var current = 0 - var timeout = false - while (runningProcess.isAlive() && !timeout) { - if (current > GENERATING_INDEX_TIMEOUT) { - java.lang.System.err - .println("Reached timeout when generating index. Terminating...") - try { - val pidOfProcess = pid(runningProcess) - val javaHome = System.getProperty("java.home") - val jstack = - if (javaHome == null) "jstack" - else - Paths.get(javaHome, "bin", "jstack").toAbsolutePath.toString - val in = java.lang.Runtime.getRuntime - .exec(Array(jstack, pidOfProcess.toString)) - .getInputStream - - System.err.println(IOUtils.toString(in, "UTF-8")) - } catch { - case e: Throwable => - java.lang.System.err - .println("Failed to get threaddump of a stuck process", e); - } finally { - timeout = true - runningProcess.destroy() + try { + val runningProcess = Process( + command, + Some(path.getAbsoluteFile.getParentFile), + "JAVA_OPTS" -> "-Dorg.jline.terminal.dumb=true" + ).run + // Poor man's solution to stuck index generation + val GENERATING_INDEX_TIMEOUT = 60 * 2 // 2 minutes + var current = 0 + var timeout = false + while (runningProcess.isAlive() && !timeout) { + if (current > GENERATING_INDEX_TIMEOUT) { + java.lang.System.err + .println( + "Reached timeout when generating index. Terminating..." + ) + try { + val pidOfProcess = pid(runningProcess) + val javaHome = System.getProperty("java.home") + val jstack = + if (javaHome == null) "jstack" + else + Paths.get(javaHome, "bin", "jstack").toAbsolutePath.toString + val in = java.lang.Runtime.getRuntime + .exec(Array(jstack, pidOfProcess.toString)) + .getInputStream + + System.err.println(IOUtils.toString(in, "UTF-8")) + } catch { + case e: Throwable => + java.lang.System.err + .println("Failed to get threaddump of a stuck process", e); + } finally { + timeout = true + runningProcess.destroy() + } + } else { + Thread.sleep(1000) + current += 1 } - } else { - Thread.sleep(1000) - current += 1 } - } - if (timeout) { - throw new RuntimeException( - s"TIMEOUT: Failed to compile $libName in $GENERATING_INDEX_TIMEOUT seconds" - ) - } - if (runningProcess.exitValue() != 0) { - throw new RuntimeException(s"Cannot compile $libName.") + if (timeout) { + throw new RuntimeException( + s"TIMEOUT: Failed to compile $libName in $GENERATING_INDEX_TIMEOUT seconds" + ) + } + if (runningProcess.exitValue() != 0) { + throw new RuntimeException(s"Cannot compile $libName.") + } + } finally { + assertExecutable("after execution") } } else { log.debug(s"No modified files. Not generating index for $libName.") @@ -302,7 +346,7 @@ object DistributionPackage { ): Boolean = { import scala.collection.JavaConverters._ - val enso = distributionRoot / "bin" / batName("enso") + val enso = distributionRoot / "bin" / batOrExeName("enso") val pb = new java.lang.ProcessBuilder() val all = new java.util.ArrayList[String]() val runArgumentIndex = locateRunArgument(args) diff --git a/project/GraalVM.scala b/project/GraalVM.scala index cea30af89bbb..28127b4e1bf1 100644 --- a/project/GraalVM.scala +++ b/project/GraalVM.scala @@ -9,6 +9,60 @@ import scala.collection.immutable.Seq /** A collection of utility methods for everything related to the GraalVM and Truffle. */ object GraalVM { + object EnsoLauncher { + val VAR_NAME = "ENSO_LAUNCHER" + + override def toString(): String = { + val prop = System.getenv(VAR_NAME) + // default value is `shell` + return if (prop == null) "shell" else prop; + } + + private lazy val parsed + : (Boolean, Boolean, Boolean, Boolean, Boolean, Boolean) = { + var shell = false + var native = false + var test = false + var debug = false + var fast = false + var disableLanguageServer = false + toString().split(",").foreach { + case "shell" => shell = true + case "native" => native = true + case "test" => { + native = true + test = true + } + case "debug" => { + native = true + debug = true + } + case "fast" => { + native = true + fast = true + } + case "-ls" => { + native = true + disableLanguageServer = true + } + case v => + throw new IllegalStateException(s"Unexpected value of $VAR_NAME: $v") + } + if (shell && native) { + throw new IllegalStateException( + s"Cannot specify `shell` and other properties in $VAR_NAME env variable" + ) + } + (shell, native, test, debug, fast, disableLanguageServer) + } + def shell = parsed._1 + def native = parsed._2 + def test = parsed._3 + def debug = parsed._4 + def fast = parsed._5 + def disableLanguageServer = parsed._6 + def release = native && !test && !debug && !fast && !disableLanguageServer + } /** Has the user requested to use Espresso for Java interop? */ private def isEspressoMode(): Boolean =