Native Language Server integration with PM (#11880)

* Native Language Server integration with PM

The first attempt at
a) building `enso` native-image that includes full Language Server
b) integrating the executable as an experimental feature during project
startup

The change (for now) assumes that `enso` executable appears in Enso's
default `engines` directory.
To build and run the new integration one has to
a) `engine-runner/buildNativeImage`
b) run PM with `--native-language-server`

This change also adds a copy of some of logback's code (`SocketAppender`
or a simple socket server`) as it was impossible to debug serialization
bugs without some additional logging.

* nit

* Add missing reflect config

Otherwise Akka and other code is simply dead-code eliminated in
native-image.

* Ensure proper Context construction

When message transport is missing, context should be properly built,
even in AOT mode.

* Add missing Akka entries in reflect-config

* nit

* Partial revert

* Fix service registration

Problems revealed after a clean build

* Fix Base_Tests in native-image

* Fix compilation issues

* Drop Truffle dependency

* fmt

* Drop TruffleOptions dependency

Despite being advertised as such, native image doesn't set
`com.oracle.graalvm.isaot` and it has to be provided separately if we
don't want to use `TruffleOptions`.

* Move NI config to LS resources

* Remove automatically added module dependency

* Move AOT Context-creation logic to factory

Context creation for AOT needs to define a separate Engine configuration
rather than using a shared engine to be able to use message transport.

* PR review

Author: hubertp

build.sbt

@@ -2283,12 +2283,13 @@ lazy val `language-server` = (project in file("engine/language-server"))
     javaModuleName := "org.enso.language.server",
     Compile / moduleDependencies ++=
       Seq(
-        "org.graalvm.polyglot"   % "polyglot"         % graalMavenPackagesVersion,
-        "org.slf4j"              % "slf4j-api"        % slf4jVersion,
-        "commons-cli"            % "commons-cli"      % commonsCliVersion,
-        "commons-io"             % "commons-io"       % commonsIoVersion,
-        "com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion,
-        "org.eclipse.jgit"       % "org.eclipse.jgit" % jgitVersion
+        "org.graalvm.polyglot"   % "polyglot"                % graalMavenPackagesVersion,
+        "org.slf4j"              % "slf4j-api"               % slf4jVersion,
+        "commons-cli"            % "commons-cli"             % commonsCliVersion,
+        "commons-io"             % "commons-io"              % commonsIoVersion,
+        "com.google.flatbuffers" % "flatbuffers-java"        % flatbuffersVersion,
+        "org.eclipse.jgit"       % "org.eclipse.jgit"        % jgitVersion,
+        "org.netbeans.api"       % "org-openide-util-lookup" % netbeansApiVersion
       ),
     Compile / internalModuleDependencies := Seq(
       (`akka-wrapper` / Compile / exportedModule).value,
@@ -3614,13 +3615,17 @@ lazy val `engine-runner` = project
       val epbLang =
         (`runtime-language-epb` / Compile / fullClasspath).value
           .map(_.data.getAbsolutePath)
+      val langServer =
+        (`language-server` / Compile / fullClasspath).value
+          .map(_.data.getAbsolutePath)
       val core = (
         runnerDeps ++
           runtimeDeps ++
           loggingDeps ++
           replDebugInstr ++
           runtimeServerInstr ++
           idExecInstr ++
+          langServer ++
           epbLang
       ).distinct
       val stdLibsJars =
@@ -3722,6 +3727,7 @@ lazy val `engine-runner` = project
               "org.jline",
               "io.methvin.watchservice",
               "zio.internal",
+              "zio",
               "org.enso.runner",
               "sun.awt",
               "sun.java2d",
@@ -3733,7 +3739,8 @@ lazy val `engine-runner` = project
               "akka.http",
               "org.enso.base",
               "org.enso.image",
-              "org.enso.table"
+              "org.enso.table",
+              "org.eclipse.jgit"
             )
           )
       }

engine/common/src/main/java/org/enso/common/ContextFactory.java

@@ -9,6 +9,7 @@
 import org.enso.logger.JulHandler;
 import org.enso.logging.config.LoggerSetup;
 import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.Engine;
 import org.graalvm.polyglot.HostAccess;
 import org.graalvm.polyglot.io.MessageTransport;
 import org.slf4j.event.Level;
@@ -52,6 +53,7 @@ public final class ContextFactory {
   private String checkForWarnings;
   private int warningsLimit = 100;
   private java.util.Map<String, String> options = new HashMap<>();
+  private String runtimerServerKey;
   private boolean enableDebugServer;
 
   private ContextFactory() {}
@@ -145,6 +147,11 @@ public ContextFactory options(Map<String, String> options) {
     return this;
   }
 
+  public ContextFactory enableRuntimeServerInfoKey(String keyName) {
+    this.runtimerServerKey = keyName;
+    return this;
+  }
+
   public ContextFactory checkForWarnings(String fqnOfMethod) {
     this.checkForWarnings = fqnOfMethod;
     return this;
@@ -161,6 +168,16 @@ public Context build() {
     }
     var julLogLevel = Converter.toJavaLevel(logLevel);
     var logLevelName = julLogLevel.getName();
+    var inAOTMode = java.lang.Boolean.getBoolean("com.oracle.graalvm.isaot");
+    java.util.Map<String, String> engineOptions = null;
+    if (runtimerServerKey != null) {
+      if (!inAOTMode) {
+        options.put(runtimerServerKey, "true");
+      } else {
+        engineOptions = new java.util.HashMap<>();
+        engineOptions.put(runtimerServerKey, "true");
+      }
+    }
     var builder =
         Context.newBuilder()
             .allowExperimentalOptions(true)
@@ -189,9 +206,6 @@ public Context build() {
     if (enableDebugServer) {
       builder.option(DebugServerInfo.ENABLE_OPTION, "true");
     }
-    if (messageTransport != null) {
-      builder.serverTransport(messageTransport);
-    }
     builder.option(RuntimeOptions.LOG_LEVEL, logLevelName);
     var logHandler = JulHandler.get();
     var logLevels = LoggerSetup.get().getConfig().getLoggers();
@@ -228,6 +242,18 @@ public Context build() {
           .allowCreateThread(true);
     }
 
+    if (inAOTMode) {
+      // In AOT mode one must not use a shared engine; the latter causes issues when initializing
+      // message transport - it is set to `null`.
+      var eng = Engine.newBuilder().allowExperimentalOptions(true).options(engineOptions);
+      if (messageTransport != null) {
+        eng.serverTransport(messageTransport);
+      }
+      builder.engine(eng.build());
+    } else if (messageTransport != null) {
+      builder.serverTransport(messageTransport);
+    }
+
     var ctx = builder.build();
     ContextInsightSetup.configureContext(ctx);
     return ctx;

engine/language-server/src/main/java/module-info.java

@@ -38,7 +38,6 @@
   requires org.enso.task.progress.notifications;
   requires org.enso.ydoc.polyfill;
 
-  exports org.enso.languageserver.boot;
   exports org.enso.languageserver.filemanager to scala.library;
   exports org.enso.languageserver.runtime to scala.library;
   exports org.enso.languageserver.search to scala.library;

engine/language-server/src/main/java/org/enso/languageserver/boot/LanguageServerRunner.java

@@ -8,6 +8,7 @@
 import org.slf4j.event.Level;
 import scala.concurrent.ExecutionContext;
 
+@org.openide.util.lookup.ServiceProvider(service = LanguageServerApi.class)
 public final class LanguageServerRunner extends LanguageServerApi {
   public LanguageServerRunner() {}
 

engine/language-server/src/main/java/org/enso/languageserver/boot/resource/RepoInitialization.java

@@ -1,15 +1,12 @@
 package org.enso.languageserver.boot.resource;
 
 import akka.event.EventStream;
-import java.io.IOException;
-import java.nio.file.FileSystemException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.util.concurrent.*;
-import org.apache.commons.io.FileUtils;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Semaphore;
 import org.enso.languageserver.data.ProjectDirectoriesConfig;
 import org.enso.languageserver.event.InitializedEvent;
-import org.enso.logger.masking.MaskedPath;
 import org.enso.searcher.memory.InMemorySuggestionsRepo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,14 +69,18 @@ public CompletableFuture<Void> init() {
   }
 
   private CompletableFuture<Void> initSuggestionsRepo() {
-    return CompletableFuture.runAsync(
-            () -> logger.debug("Initializing suggestions repo [{}]...", suggestionsRepo), executor)
-        .thenComposeAsync(
-            v -> {
-              if (!isInitialized)
-                return doInitSuggestionsRepo()
-                    .exceptionallyComposeAsync(this::recoverInitializationError, executor);
-              else return CompletableFuture.completedFuture(v);
+    return CompletableFuture.supplyAsync(
+            () -> {
+              logger.debug("Initializing Suggestions repo [{}]...", suggestionsRepo);
+              try {
+                lock.acquire();
+                if (!isInitialized)
+                  return doInitSuggestionsRepo()
+                      .exceptionallyComposeAsync(this::recoverInitializationError, executor);
+                else return CompletableFuture.completedFuture(null);
+              } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+              }
             },
             executor)
         .thenRunAsync(
@@ -105,60 +106,6 @@ private CompletableFuture<Void> recoverInitializationError(Throwable error) {
         .thenComposeAsync(v -> doInitSuggestionsRepo(), executor);
   }
 
-  private CompletableFuture<Void> clearDatabaseFile(int retries) {
-    return CompletableFuture.runAsync(
-            () -> {
-              if (!isInitialized) {
-                logger.debug("Clear database file. Attempt #{}", retries + 1);
-                try {
-                  Files.delete(projectDirectoriesConfig.suggestionsDatabaseFile().toPath());
-                } catch (IOException e) {
-                  throw new CompletionException(e);
-                }
-              }
-            },
-            executor)
-        .exceptionallyComposeAsync(error -> recoverClearDatabaseFile(error, retries), executor);
-  }
-
-  private CompletableFuture<Void> recoverClearDatabaseFile(Throwable error, int retries) {
-    if (error instanceof CompletionException) {
-      return recoverClearDatabaseFile(error.getCause(), retries);
-    } else if (error instanceof NoSuchFileException) {
-      logger.warn(
-          "Failed to delete the database file. Attempt #{}. File does not exist [{}]",
-          retries + 1,
-          new MaskedPath(projectDirectoriesConfig.suggestionsDatabaseFile().toPath()));
-      return CompletableFuture.completedFuture(null);
-    } else if (error instanceof FileSystemException) {
-      logger.error(
-          "Failed to delete the database file. Attempt #{}. The file will be removed during the"
-              + " shutdown",
-          retries + 1,
-          error);
-      Runtime.getRuntime()
-          .addShutdownHook(
-              new Thread(
-                  () ->
-                      FileUtils.deleteQuietly(projectDirectoriesConfig.suggestionsDatabaseFile())));
-      return CompletableFuture.failedFuture(error);
-    } else if (error instanceof IOException) {
-      logger.error("Failed to delete the database file. Attempt #{}", retries + 1, error);
-      if (retries < MAX_RETRIES) {
-        try {
-          Thread.sleep(RETRY_DELAY_MILLIS);
-        } catch (InterruptedException e) {
-          throw new CompletionException(e);
-        }
-        return clearDatabaseFile(retries + 1);
-      } else {
-        return CompletableFuture.failedFuture(error);
-      }
-    }
-
-    return CompletableFuture.completedFuture(null);
-  }
-
   private CompletionStage<Void> doInitSuggestionsRepo() {
     return FutureConverters.asJava(suggestionsRepo.init()).thenAcceptAsync(res -> {}, executor);
   }