[JS] Add GenAI Node.js bindings

CMakeLists.txt

@@ -109,6 +109,9 @@ set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_g
 if(ENABLE_PYTHON)
     list(APPEND CPACK_COMPONENTS_ALL pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR})
 endif()
+if(ENABLE_JS)
+    list(APPEND CPACK_COMPONENTS_ALL genai_node_addon)
+endif()
 if(WIN32 AND NOT DEFINED CPACK_GENERATOR)
     set(CPACK_GENERATOR "ZIP")
 endif()

cmake/features.cmake

@@ -3,3 +3,4 @@
 #
 
 option(ENABLE_PYTHON "Enable Python API build" ON)
+option(ENABLE_JS "Enable JS API build" OFF)

samples/CMakeLists.txt

@@ -2,6 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+# Samples do not need to be built for NPM package
+if(CPACK_GENERATOR STREQUAL "NPM")
+    return()
+endif()
+
 add_subdirectory(cpp/beam_search_causal_lm)
 add_subdirectory(cpp/benchmark_genai)
 add_subdirectory(cpp/chat_sample)

samples/js/chat_sample/.gitignore

@@ -0,0 +1 @@
+node_modules

samples/js/chat_sample/README.md

@@ -0,0 +1,48 @@
+# JavaScript chat_sample that supports most popular models like LLaMA 3
+
+This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample fearures `Pipeline.LLMPipeline` and configures it for the chat scenario.
+
+## Download and convert the model and tokenizers
+
+To convert model you have to use python package `optimum-intel`.
+The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version.
+
+Install [../../export-requirements.txt](../../export-requirements.txt) to convert a model.
+
+```sh
+pip install --upgrade-strategy eager -r ../../export-requirements.txt
+optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
+```
+
+## Run:
+
+Compile GenAI JavaScript bindings archive first using the instructions in [../../../src/js/README.md](../../../src/js/README.md#build-bindings).
+
+Run `npm install` in current folder and then run the sample:
+
+`node chat_sample.js TinyLlama-1.1B-Chat-v1.0`
+
+Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is recommended to run larger models on a dGPU with 32GB+ RAM. For example, the model meta-llama/Llama-2-13b-chat-hf can benefit from being run on a dGPU. Modify the source code to change the device for inference to the GPU.
+
+See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models.
+
+### Troubleshooting
+
+#### Unicode characters encoding error on Windows
+
+Example error:
+```
+UnicodeEncodeError: 'charmap' codec can't encode character '\u25aa' in position 0: character maps to <undefined>
+```
+
+If you encounter the error described in the example when sample is printing output to the Windows console, it is likely due to the default Windows encoding not supporting certain Unicode characters. To resolve this:
+1. Enable Unicode characters for Windows cmd - open `Region` settings from `Control panel`. `Administrative`->`Change system locale`->`Beta: Use Unicode UTF-8 for worldwide language support`->`OK`. Reboot.
+2. Enable UTF-8 mode by setting environment variable `PYTHONIOENCODING="utf8"`.
+
+#### Missing chat template
+
+If you encounter an exception indicating a missing "chat template" when launching the `ov::genai::LLMPipeline` in chat mode, it likely means the model was not tuned for chat functionality. To work this around, manually add the chat template to tokenizer_config.json of your model.
+The following template can be used as a default, but it may not work properly with every model:
+```
+"chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n<|im_start|>assistant\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>\n'}}{% endif %}{% endfor %}",
+```

samples/js/chat_sample/chat_sample.js

@@ -0,0 +1,54 @@
+import readline from 'readline';
+import { Pipeline } from 'genai-node';
+
+main();
+
+function streamer(subword) {
+  process.stdout.write(subword);
+}
+
+async function main() {
+  const MODEL_PATH = process.argv[2];
+
+  if (!MODEL_PATH) {
+    console.error('Please specify path to model directory\n'
+                  + 'Run command must be: `node chat_sample.js *path_to_model_dir*`');
+    process.exit(1);
+  }
+
+  const device = 'CPU'; // GPU can be used as well
+
+  // Create interface for reading user input from stdin
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const pipe = await Pipeline.LLMPipeline(MODEL_PATH, device);
+  const config = { 'max_new_tokens': 100 };
+
+  await pipe.startChat();
+  promptUser();
+
+  // Function to prompt the user for input
+  function promptUser() {
+    rl.question('question:\n', handleInput);
+  }
+
+  // Function to handle user input
+  async function handleInput(input) {
+    input = input.trim();
+
+    // Check for exit command
+    if (!input) {
+      await pipe.finishChat();
+      rl.close();
+      process.exit(0);
+    }
+
+    await pipe.generate(input, config, streamer);
+    console.log('\n----------');
+
+    if (!rl.closed) promptUser();
+  }
+}

samples/js/chat_sample/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "genai-node-demo",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "type": "module",
+  "devDependencies": {
+    "genai-node": "../../../src/js/"
+  },
+  "engines": {
+    "node": ">=21.0.0"
+  },
+  "scripts": {
+    "test": "node tests/usage.test.js"
+  }
+}

samples/js/chat_sample/tests/usage.test.js

@@ -0,0 +1,63 @@
+import { env } from 'process';
+import { spawn } from 'child_process';
+
+const MODEL_PATH = env.MODEL_PATH;
+const prompt = 'Tell me exactly, no changes, print as is: "Hello world"';
+const expected = 'Hello world';
+
+if (!MODEL_PATH)
+  throw new Error(
+    'Please environment variable MODEL_PATH to the path of the model directory'
+  );
+
+const runTest = async () => {
+  return new Promise((resolve, reject) => {
+    const script = spawn('node', ['chat_sample.js', MODEL_PATH]);
+    let output = '';
+
+    // Collect output from stdout
+    script.stdout.on('data', (data) => {
+      output += data.toString();
+    });
+
+    // Capture errors
+    script.stderr.on('data', (data) => {
+      reject(data.toString());
+    });
+
+    // Send input after detecting the question prompt
+    script.stdout.once('data', (data) => {
+      if (data.toString().startsWith('question:')) {
+        script.stdin.write(`${prompt}\n`); // Provide input
+        script.stdin.end(); // Close stdin to signal EOF
+      }
+    });
+
+    // Check results when the process exits
+    script.on('close', (code) => {
+      if (code !== 0) {
+        return reject(`Process exited with code ${code}`);
+      }
+
+      // Log the output
+      console.log(`Result output: ${output}`);
+
+      // Validate the output
+      if (output.includes(expected)) {
+        resolve('Test passed!');
+      } else {
+        reject('Test failed: Output did not match expected result.');
+      }
+    });
+  });
+};
+
+runTest()
+  .then((message) => {
+    console.log(message);
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

src/CMakeLists.txt

@@ -7,3 +7,7 @@ add_subdirectory(cpp)
 if(ENABLE_PYTHON)
     add_subdirectory(python)
 endif()
+
+if(ENABLE_JS)
+    add_subdirectory(js)
+endif()

src/cpp/CMakeLists.txt

@@ -147,13 +147,42 @@ if(MSVC OR APPLE)
     set(ARCH_DIR ${ARCH_DIR}/${CMAKE_BUILD_TYPE})
 endif()
 
+# Put binaries at the top level for NPM package
+if(CPACK_GENERATOR STREQUAL "NPM")
+    set(LIBRARY_DESTINATION .)
+    set(ARCHIVE_DESTINATION .)
+    set(RUNTIME_DESTINATION .)
+
+    # setting RPATH / LC_RPATH depending on platform
+    if(LINUX)
+        # to find libopenvino.so in the same folder
+        set(rpaths "$ORIGIN")
+    elseif(APPLE)
+        # to find libopenvino.dylib in the same folder
+        set(rpaths "@loader_path")
+    endif()
+
+    if(rpaths)
+        set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}")
+    endif()
+else()
+    set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR})
+    set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR})
+    set(RUNTIME_DESTINATION runtime/bin/${ARCH_DIR})
+endif()
+
 install(TARGETS ${TARGET_NAME} EXPORT OpenVINOGenAITargets
-        LIBRARY DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai
+        LIBRARY DESTINATION ${LIBRARY_DESTINATION} COMPONENT core_genai
             NAMELINK_COMPONENT core_genai_dev
-        ARCHIVE DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai_dev
-        RUNTIME DESTINATION runtime/bin/${ARCH_DIR} COMPONENT core_genai
+        ARCHIVE DESTINATION ${ARCHIVE_DESTINATION} COMPONENT core_genai_dev
+        RUNTIME DESTINATION ${RUNTIME_DESTINATION} COMPONENT core_genai
         INCLUDES DESTINATION runtime/include)
 
+# samples do not need to be built for NPM package
+if(CPACK_GENERATOR STREQUAL "NPM")
+    return()
+endif()
+
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/
         DESTINATION runtime/include COMPONENT core_genai_dev)
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvino/genai/version.hpp

src/js/.gitignore

@@ -0,0 +1,7 @@
+.vscode
+bin
+bin.*
+build
+thirdparty
+node_modules
+tests/models

src/js/.npmignore

@@ -0,0 +1,15 @@
+.vscode
+bin.*
+build
+include
+src
+tests
+
+.eslintrc.js
+CMakeLists.txt
+tsconfig.json
+TODO.md
+build.sh
+
+**/*.tsbuildinfo
+*.tgz