Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ENSO_LAUNCHER can be native,test,debug,fast,-ls #12117

Merged
merged 15 commits into from
Jan 24, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 92 additions & 40 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -3691,9 +3691,19 @@ lazy val `engine-runner` = project
val epbLang =
(`runtime-language-epb` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
val langServer =
(`language-server` / Compile / fullClasspath).value
def langServer = {
val log = streams.value.log
val path = (`language-server` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
if (GraalVM.EnsoLauncher.disableLanguageServer) {
log.info(
s"Skipping language server in native image build as ${GraalVM.EnsoLauncher.VAR_NAME} env variable is ${GraalVM.EnsoLauncher.toString}"
)
Seq()
} else {
path
}
}
val core = (
runnerDeps ++
runtimeDeps ++
Expand Down Expand Up @@ -3822,7 +3832,7 @@ lazy val `engine-runner` = project
.dependsOn(NativeImage.additionalCp)
.dependsOn(NativeImage.smallJdk)
.dependsOn(
createEnginePackage
createEnginePackageNoIndex
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
)
.value,
buildNativeImage := Def.taskDyn {
Expand Down Expand Up @@ -5106,6 +5116,7 @@ lazy val createEnginePackage =
taskKey[Unit]("Creates the engine distribution package")
createEnginePackage := {
updateLibraryManifests.value
buildEngineDistributionNoIndex.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
val log = streams.value.log
Expand All @@ -5131,48 +5142,49 @@ ThisBuild / createEnginePackage := {
createEnginePackage.result.value
}

lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution and optionally native image")
buildEngineDistribution := Def.taskIf {
lazy val createEnginePackageNoIndex =
taskKey[Unit]("Creates the engine distribution package")
createEnginePackageNoIndex := {
updateLibraryManifests.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
DistributionPackage.createEnginePackage(
distributionRoot = root,
cacheFactory = cacheFactory,
log = log,
jarModulesToCopy = modulesToCopy,
graalVersion = graalMavenPackagesVersion,
javaVersion = graalVersion,
ensoVersion = ensoVersion,
editionName = currentEdition,
sourceStdlibVersion = stdLibVersion,
targetStdlibVersion = targetStdlibVersion,
targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value,
generateIndex = false
)
log.info(s"Engine package created at $root")
}

ThisBuild / createEnginePackageNoIndex := {
createEnginePackageNoIndex.result.value
}

lazy val buildEngineDistributionNoIndex =
taskKey[Unit](
"Builds the engine distribution without generating indexes and optionally generating native image"
)
buildEngineDistributionNoIndex := Def.taskIf {
createEnginePackageNoIndex.value
if (shouldBuildNativeImage.value) {
createEnginePackage.value
(`engine-runner` / buildNativeImage).value
} else {
createEnginePackage.value
}
}.value

// This makes the buildEngineDistribution task usable as a dependency
// of other tasks.
ThisBuild / buildEngineDistribution := {
buildEngineDistribution.result.value
}

lazy val shouldBuildNativeImage = taskKey[Boolean](
"Whether native image should be build within buildEngineDistribution task"
)

ThisBuild / shouldBuildNativeImage := {
val prop = System.getenv("ENSO_LAUNCHER")
prop == "native" || prop == "debugnative"
}

ThisBuild / NativeImage.additionalOpts := {
val prop = System.getenv("ENSO_LAUNCHER")
if (prop == "native") {
Seq("-O3")
} else {
Seq("-ea", "-Ob", "-H:GenerateDebugInfo=1")
}
}

ThisBuild / engineDistributionRoot := {
engineDistributionRoot.value
}

lazy val buildEngineDistributionNoIndex =
taskKey[Unit]("Builds the engine distribution without generating indexes")
buildEngineDistributionNoIndex := {
ThisBuild / buildEngineDistributionNoIndex := {
updateLibraryManifests.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
Expand All @@ -5195,10 +5207,49 @@ buildEngineDistributionNoIndex := {
log.info(s"Engine package created at $root")
}

lazy val shouldBuildNativeImage = taskKey[Boolean](
"Whether native image should be build within buildEngineDistribution task"
)

ThisBuild / shouldBuildNativeImage := {
GraalVM.EnsoLauncher.native
}

ThisBuild / NativeImage.additionalOpts := {
if (GraalVM.EnsoLauncher.shell) {
Seq()
} else {
var opts = if (GraalVM.EnsoLauncher.release) {
Seq("-O3")
} else {
Seq("-Ob")
}

if (GraalVM.EnsoLauncher.debug) {
opts = opts ++ Seq("-H:GenerateDebugInfo=1")
}
if (GraalVM.EnsoLauncher.test) {
opts = opts ++ Seq("-ea")
}
opts
}
}

ThisBuild / engineDistributionRoot := {
engineDistributionRoot.value
}

lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution")
buildEngineDistribution := {
buildEngineDistributionNoIndex.value
createEnginePackage.value
}

// This makes the buildEngineDistributionNoIndex task usable as a dependency
// of other tasks.
ThisBuild / buildEngineDistributionNoIndex := {
buildEngineDistributionNoIndex.result.value
ThisBuild / buildEngineDistribution := {
buildEngineDistribution.result.value
}

lazy val runEngineDistribution =
Expand Down Expand Up @@ -5284,6 +5335,7 @@ buildStdLib := Def.inputTaskDyn {

lazy val pkgStdLibInternal = inputKey[Unit]("Use `buildStdLib`")
pkgStdLibInternal := Def.inputTask {
buildEngineDistributionNoIndex.value
val cmd = allStdBits.parsed
val root = engineDistributionRoot.value
val log: sbt.Logger = streams.value.log
Expand Down
34 changes: 12 additions & 22 deletions build_tools/build/src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,43 +180,33 @@ pub enum EngineLauncher {
/// The binary inside the engine distribution will be built as an optimized native image
Native,
/// The binary inside the engine distribution will be built as native image with assertions
/// enabled but no debug information
TestNative,
/// The binary inside the engine distribution will be built as native image with assertions
/// enabled and debug information
DebugNative,
TestDebugNative,
/// The binary inside the engine distribution will be a shell script
#[default]
Shell,
}

impl FromStr for EngineLauncher {
type Err = anyhow::Error;

fn from_str(s: &str) -> Result<Self> {
match s {
"native" => Ok(Self::Native),
"debugnative" => Ok(Self::DebugNative),
"shell" => Ok(Self::Shell),
_ => bail!("Invalid Engine Launcher type: {}", s),
}
bail!("Parsing of ENSO_LAUNCHER isn't needed: {}", s)
}
}

impl From<EngineLauncher> for String {
fn from(value: EngineLauncher) -> Self {
match value {
impl Display for EngineLauncher {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
let str = match self {
EngineLauncher::Native => "native".to_string(),
EngineLauncher::DebugNative => "debugnative".to_string(),
EngineLauncher::TestNative => "native,test".to_string(),
EngineLauncher::TestDebugNative => "native,test,debug".to_string(),
EngineLauncher::Shell => "shell".to_string(),
}
}
}
};

impl Display for EngineLauncher {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match *self {
EngineLauncher::Native => write!(f, "native"),
EngineLauncher::DebugNative => write!(f, "debugnative"),
EngineLauncher::Shell => write!(f, "shell"),
}
write!(f, "{}", str)
}
}

Expand Down
2 changes: 1 addition & 1 deletion build_tools/build/src/engine/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ impl RunContext {
sbt.call_arg("syntax-rust-definition/Runtime/managedClasspath").await?;
}
if self.config.build_native_runner {
env::ENSO_LAUNCHER.set(&engine::EngineLauncher::DebugNative)?;
env::ENSO_LAUNCHER.set(&engine::EngineLauncher::TestNative)?;
}

// TODO: Once the native image is production ready, we should switch to
Expand Down
33 changes: 19 additions & 14 deletions docs/infrastructure/native-image.md
Original file line number Diff line number Diff line change
Expand Up @@ -206,25 +206,30 @@ one of the following:

- `shell`: The default value. `buildEngineDistribution` command does not build
the native image.
- `debugnative`: `buildEngineDistribution` command builds native image with
assertions enabled (`-ea`). Useful for running tests on the CI.
- `native`: `buildEngineDistribution` command builds native image with
assertions disabled (`-ea`). Turns on maximal optimizations which may increase
the build time.

To generate the Native Image for runner either explicitly execute

```bash
sbt> engine-runner/buildNativeImage
```

or
- `native`: `buildEngineDistribution` command builds native image in _release
mode_ - e.g. turns on maximal optimizations increasing the build time.
- There are additional variants of `native` useful for _development_. They are
specified as comma separated attributes following `native`:
- using `native,fast` turns on _native image_ build, but disables
optimizations - e.g. produces build similar to _release mode_, but more
quickly
- using `native,test` _enables assertions_ - e.g. it instructs
`buildEngineDistribution` command to build native image with assertions
enabled (`-ea`). Useful for running Enso tests in the _native mode_.
- using `native,debug` generates _debugging informations_ for VSCode _native
image debugger_
- using `native,-ls` disables support for _language server_ in the generated
binary
- it is possible to combine all features - e.g. use `debug,fast,test,native`

To test _native image_ launcher choose one of the `native` configurations and
invoke:

```bash
$ ENSO_LAUNCHER=native sbt buildEngineDistribution
```

and execute any program with that binary - for example `test/Base_Tests`
then execute any program with that binary - for example `test/Base_Tests`

```bash
$ ./built-distribution/enso-engine-*/enso-*/bin/enso --run test/Base_Tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.LinkedHashSet;
import java.util.TreeSet;
import org.enso.compiler.core.EnsoParser;
Expand Down Expand Up @@ -110,7 +111,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs());
for (var nativeLib : nativeLibs) {
var out = new File(nativeLibDir, nativeLib.getName());
Files.copy(nativeLib.toPath(), out.toPath());
Files.copy(nativeLib.toPath(), out.toPath(), StandardCopyOption.REPLACE_EXISTING);
nativeLibPaths.add(out.getAbsolutePath());
}
}
Expand Down
Loading
Loading