Port clangd browser example to here

- WIP: Build container images with GHA
TypeFox · Aug 23, 2024 · 62e7c77 · 62e7c77
1 parent d5b563b
commit 62e7c77
Show file tree

Hide file tree

Showing 25 changed files with 786 additions and 4 deletions.
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
@@ -0,0 +1,62 @@
+name: Build container images
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME_JDT: ${{ github.repository }}/eclipse-jdt-ls
+
+jobs:
+  images:
+    name: container images build and deploy
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    strategy:
+        matrix:
+         version: [ 10, 12]
+         os: [ ubuntu-latest, windows-latest, macos-latest ]
+
+    timeout-minutes: 60
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta_jdt
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_JDT }}
+        # enforce latest tag for now
+        tags: |
+          type=raw,value=latest
+
+    - name: Build and push Docker image
+      id: push_jdt
+      uses: docker/build-push-action@v5
+      with:
+        context: ./packages/examples/resources/eclipse.jdt.ls
+        push: true
+        tags: ${{ steps.meta_jdt.outputs.tags }}
+        labels: ${{ steps.meta_jdt.outputs.labels }}
+
+    - name: Generate artifact attestation
+      uses: actions/attest-build-provenance@v1
+      with:
+        subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_JDT }}
+        subject-digest: ${{ steps.push_jdt.outputs.digest }}
+        push-to-registry: true
diff --git a/index.html b/index.html
@@ -54,6 +54,11 @@ <h3>Groovy</h3>
         <a href="./packages/examples/groovy.html">Groovy Language Client & Language Server (Web Socket)</a>
         <br>
 
+        <h3>Cpp / Clangd</h3>
+        <a href="./packages/examples/clangd.html">Cpp Language Client & Clangd Language Server (Worker/Wasm)</a>
+        <br>
+
+
         <h2>Monaco Editor React</h2>
         <a href="./packages/examples/react_statemachine.html">React: Langium Statemachine Language Client & Language Server (Worker)</a>
         <br>

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/examples/.gitignore b/packages/examples/.gitignore
@@ -1,3 +1,4 @@
+resources/clangd/wasm
 resources/groovy/external
 resources/eclipse.jdt.ls/ls
 resources/eclipse.jdt.ls/*.tar.gz

diff --git a/packages/examples/clangd.html b/packages/examples/clangd.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <title>Cpp Language Client & Clangd Language Server (Worker/Wasm)</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="style.css">
+</head>
+
+<body>
+	<h2>Cpp Language Client & Clangd Language Server (Worker/Wasm)</h2>
+    <button type="button" id="button-start">Start</button>
+    <button type="button" id="button-dispose">Dispose</button>
+	<div id="monaco-editor-root" style="width:800px;height:600px;border:1px solid grey"></div>
+    <script type="module">
+        import { configureMonacoWorkers, runClangdWrapper } from "./src/clangd/client/main.ts";
+
+        configureMonacoWorkers();
+        runClangdWrapper();
+    </script>
+</body>
+
+</html>
diff --git a/packages/examples/package.json b/packages/examples/package.json
@@ -55,6 +55,7 @@
   },
   "dependencies": {
     "@codingame/monaco-vscode-configuration-service-override": "~8.0.2",
+    "@codingame/monaco-vscode-cpp-default-extension": "~8.0.2",
     "@codingame/monaco-vscode-files-service-override": "~8.0.2",
     "@codingame/monaco-vscode-groovy-default-extension": "~8.0.2",
     "@codingame/monaco-vscode-keybindings-service-override": "~8.0.2",
@@ -121,6 +122,7 @@
     "start:server:python": "vite-node src/python/server/direct.ts",
     "start:server:groovy": "vite-node src/groovy/server/direct.ts",
     "start:server:jdtls": "vite-node src/eclipse.jdt.ls/server/direct.ts",
-    "langium:generate": "langium generate --file ./src/langium/statemachine/config/langium-config.json"
+    "langium:generate": "langium generate --file ./src/langium/statemachine/config/langium-config.json",
+    "extract:docker": "vite-node ./resources/clangd/scripts/extractDockerFiles.ts"
   }
 }
diff --git a/packages/examples/resources/clangd/build-docker.sh b/packages/examples/resources/clangd/build-docker.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# 3b. Build LLVM
+
+WORKSPACE_DIR=$PWD
+# otherwise emcmake is not found
+source $WORKSPACE_DIR/emsdk/emsdk_env.sh
+
+cd llvm-project
+
+## Build native tools first
+cmake -G Ninja -S llvm -B build-native \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_PROJECTS=clang
+cmake --build build-native --target llvm-tblgen clang-tblgen
+
+## Apply a patch for blocking stdin read
+git apply $WORKSPACE_DIR/wait_stdin.patch
+
+## Build clangd (1st time, just for compiler headers)
+emcmake cmake -G Ninja -S llvm -B build \
+    -DCMAKE_CXX_FLAGS="-pthread -Dwait4=__syscall_wait4" \
+    -DCMAKE_EXE_LINKER_FLAGS="-pthread -s ENVIRONMENT=worker -s NO_INVOKE_RUN" \
+    -DCMAKE_BUILD_TYPE=MinSizeRel \
+    -DLLVM_TARGET_ARCH=wasm32-emscripten \
+    -DLLVM_DEFAULT_TARGET_TRIPLE=wasm32-wasi \
+    -DLLVM_TARGETS_TO_BUILD=WebAssembly \
+    -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" \
+    -DLLVM_TABLEGEN=$PWD/build-native/bin/llvm-tblgen \
+    -DCLANG_TABLEGEN=$PWD/build-native/bin/clang-tblgen \
+    -DLLVM_BUILD_STATIC=ON \
+    -DLLVM_INCLUDE_EXAMPLES=OFF \
+    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_ENABLE_BACKTRACES=OFF \
+    -DLLVM_ENABLE_UNWIND_TABLES=OFF \
+    -DLLVM_ENABLE_CRASH_OVERRIDES=OFF \
+    -DCLANG_ENABLE_STATIC_ANALYZER=OFF \
+    -DLLVM_ENABLE_TERMINFO=OFF \
+    -DLLVM_ENABLE_PIC=OFF \
+    -DLLVM_ENABLE_ZLIB=OFF \
+    -DCLANG_ENABLE_ARCMT=OFF
+cmake --build build --target clangd
+
+## Copy installed headers to WASI sysroot
+cp -r build/lib/clang/$LLVM_VER_MAJOR/include/* $WORKSPACE_DIR/wasi-sysroot/include/
+
+## Build clangd (2nd time, for the real thing)
+emcmake cmake -G Ninja -S llvm -B build \
+    -DCMAKE_CXX_FLAGS="-pthread -Dwait4=__syscall_wait4" \
+    -DCMAKE_EXE_LINKER_FLAGS="-pthread -s ENVIRONMENT=worker -s NO_INVOKE_RUN -s EXIT_RUNTIME -s INITIAL_MEMORY=2GB -s ALLOW_MEMORY_GROWTH -s MAXIMUM_MEMORY=4GB -s STACK_SIZE=256kB -s EXPORTED_RUNTIME_METHODS=FS,callMain -s MODULARIZE -s EXPORT_ES6 -s WASM_BIGINT -s ASSERTIONS -s ASYNCIFY -s PTHREAD_POOL_SIZE='Math.max(navigator.hardwareConcurrency, 8)' --embed-file=$WORKSPACE_DIR/wasi-sysroot/include@/usr/include" \
+    -DCMAKE_BUILD_TYPE=MinSizeRel \
+    -DLLVM_TARGET_ARCH=wasm32-emscripten \
+    -DLLVM_DEFAULT_TARGET_TRIPLE=wasm32-wasi \
+    -DLLVM_TARGETS_TO_BUILD=WebAssembly \
+    -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" \
+    -DLLVM_TABLEGEN=$PWD/build-native/bin/llvm-tblgen \
+    -DCLANG_TABLEGEN=$PWD/build-native/bin/clang-tblgen \
+    -DLLVM_BUILD_STATIC=ON \
+    -DLLVM_INCLUDE_EXAMPLES=OFF \
+    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_ENABLE_BACKTRACES=OFF \
+    -DLLVM_ENABLE_UNWIND_TABLES=OFF \
+    -DLLVM_ENABLE_CRASH_OVERRIDES=OFF \
+    -DCLANG_ENABLE_STATIC_ANALYZER=OFF \
+    -DLLVM_ENABLE_TERMINFO=OFF \
+    -DLLVM_ENABLE_PIC=OFF \
+    -DLLVM_ENABLE_ZLIB=OFF \
+    -DCLANG_ENABLE_ARCMT=OFF
+cmake --build build --target clangd
diff --git a/packages/examples/resources/clangd/build.Dockerfile b/packages/examples/resources/clangd/build.Dockerfile
@@ -0,0 +1,6 @@
+FROM clangd-clangd-configure
+
+COPY build-docker.sh /builder/build-docker.sh
+COPY wait_stdin.patch /builder/wait_stdin.patch
+
+RUN (cd /builder; ./build-docker.sh)
diff --git a/packages/examples/resources/clangd/build.docker-compose.yml b/packages/examples/resources/clangd/build.docker-compose.yml
@@ -0,0 +1,10 @@
+services:
+  clangd-build:
+    build:
+      dockerfile: ./build.Dockerfile
+      context: .
+      # only linux/amd64 for now
+      platforms:
+        - "linux/amd64"
+    platform: linux/amd64
+    container_name: clangd-build
diff --git a/packages/examples/resources/clangd/configure-docker.sh b/packages/examples/resources/clangd/configure-docker.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# It's not recommend for you to run this script directly,
+# (because I'm not good at writing this sorry)
+# but you can use it as a reference for building.
+
+# 0. Configs
+
+# sudo apt install vim git build-essential cmake ninja-build python3
+
+## Note: Better to make sure WASI SDK version matches the LLVM version
+EMSDK_VER=3.1.52
+WASI_SDK_VER=22.0
+WASI_SDK_VER_MAJOR=22
+LLVM_VER=18.1.2
+LLVM_VER_MAJOR=18
+
+# 1. Get Emscripten
+
+git clone --branch $EMSDK_VER --depth 1 https://github.com/emscripten-core/emsdk
+pushd emsdk
+./emsdk install $EMSDK_VER
+./emsdk activate $EMSDK_VER
+source ./emsdk_env.sh
+popd
+
+# 2. Prepare WASI sysroot
+
+wget -O- https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_SDK_VER_MAJOR/wasi-sysroot-$WASI_SDK_VER.tar.gz | tar -xz
+
+# 3a. Build LLVM
+
+git clone --branch llvmorg-$LLVM_VER --depth 1 https://github.com/llvm/llvm-project
diff --git a/packages/examples/resources/clangd/configure.Dockerfile b/packages/examples/resources/clangd/configure.Dockerfile
@@ -0,0 +1,18 @@
+FROM ubuntu
+
+RUN apt update \
+    && apt upgrade -y \
+    && apt install -y curl git wget build-essential cmake ninja-build python3
+
+RUN curl https://get.volta.sh | bash
+ENV VOLTA_FEATURE_PNPM=1
+ENV VOLTA_HOME "/root/.volta"
+ENV PATH "$VOLTA_HOME/bin:$PATH"
+
+RUN volta install node \
+    && volta install pnpm
+
+RUN mkdir /builder
+
+COPY configure-docker.sh /builder/configure-docker.sh
+RUN (cd /builder; ./configure-docker.sh)
diff --git a/packages/examples/resources/clangd/configure.docker-compose.yml b/packages/examples/resources/clangd/configure.docker-compose.yml
@@ -0,0 +1,10 @@
+services:
+  clangd-configure:
+    build:
+      dockerfile: ./configure.Dockerfile
+      context: .
+      # only linux/amd64 for now
+      platforms:
+        - "linux/amd64"
+    platform: linux/amd64
+    container_name: clangd-configure
diff --git a/packages/examples/resources/clangd/scripts/extractDockerFiles.ts b/packages/examples/resources/clangd/scripts/extractDockerFiles.ts
@@ -0,0 +1,18 @@
+import * as fs from "node:fs";
+import child_process from "node:child_process";
+
+const outputDir = './resources/clangd/wasm';
+
+// clean always
+fs.rmSync(outputDir, {
+    force: true,
+    recursive: true
+});
+fs.mkdirSync(outputDir);
+
+child_process.execFileSync('docker', ['create', '--name', 'extract-clangd', 'clangd-clangd-build']);
+child_process.execFileSync('docker', ['cp', 'extract-clangd:/builder/llvm-project/build/bin/clangd.js', outputDir]);
+child_process.execFileSync('docker', ['cp', 'extract-clangd:/builder/llvm-project/build/bin/clangd.wasm', outputDir]);
+child_process.execFileSync('docker', ['cp', 'extract-clangd:/builder/llvm-project/build/bin/clangd.worker.js', outputDir]);
+child_process.execFileSync('docker', ['cp', 'extract-clangd:/builder/llvm-project/build/bin/clangd.worker.mjs', outputDir]);
+child_process.execFileSync('docker', ['rm', 'extract-clangd']);
diff --git a/packages/examples/resources/clangd/wait_stdin.patch b/packages/examples/resources/clangd/wait_stdin.patch
@@ -0,0 +1,31 @@
+diff --git a/clang-tools-extra/clangd/JSONTransport.cpp b/clang-tools-extra/clangd/JSONTransport.cpp
+index 9dc0df807..b1a4e9bd1 100644
+--- a/clang-tools-extra/clangd/JSONTransport.cpp
++++ b/clang-tools-extra/clangd/JSONTransport.cpp
+@@ -1,3 +1,26 @@
++#ifdef __EMSCRIPTEN__
++
++#include <emscripten.h>
++
++#include "support/Shutdown.h"
++#include <utility>
++
++EM_ASYNC_JS(void, waitForStdin, (), {
++  await Module.stdinReady();
++})
++
++template <typename Fun, typename Ret = decltype(std::declval<Fun>()())>
++Ret doUntilStdinAvailable(
++    const std::enable_if_t<true, Ret>& fail,
++    const Fun& f) {
++  waitForStdin();
++  return clang::clangd::retryAfterSignalUnlessShutdown(fail, f);
++}
++
++#define retryAfterSignalUnlessShutdown doUntilStdinAvailable
++
++#endif
++
+ //===--- JSONTransport.cpp - sending and receiving LSP messages over JSON -===//
+ //
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
diff --git a/packages/examples/src/clangd/README.md b/packages/examples/src/clangd/README.md
@@ -0,0 +1,5 @@
+# Cpp Language Client & Clangd Language Server (Worker/Wasm)
+
+This example is based on the wonderful [work](https://github.com/guyutongxue/clangd-in-browser) from [Guyutongxue](https://github.com/guyutongxue).
+
+This example has less features and therefore less code and will be used to test and integrate new features under development.