#167: Added Exasol Text-AI Installer Wrapper (#178)

* #168: Installed Span-UDFs in transformers_extension_wrapper 
* Change poetry version in actions to 2.0.1
* Split slow checks into SaaS and Text-AI checks with larger runners

---------

Co-authored-by: Torsten Kilias <tkilias@users.noreply.github.com>
Co-authored-by: Christoph Kuhnke <github@kuhnke.net>

Author: 3 people

.github/workflows/build-and-publish.yml

@@ -17,7 +17,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Build Artifacts
         run: poetry build

.github/workflows/check-release-tag.yml

@@ -14,7 +14,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Check Tag Version
         # make sure the pushed/created tag matched the project version

.github/workflows/checks.yml

@@ -19,7 +19,9 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Check Version(s)
         run: poetry run version-check version.py
@@ -34,7 +36,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Build Documentation
         run: |
@@ -54,8 +58,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
         with:
+          poetry-version: 2.0.1
           python-version: ${{ matrix.python-version }}
 
       - name: Run lint
@@ -82,8 +87,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
         with:
+          poetry-version: 2.0.1
           python-version: ${{ matrix.python-version }}
 
       - name: Run type-check
@@ -98,8 +104,9 @@ jobs:
        - name: SCM Checkout
          uses: actions/checkout@v4
        - name: Setup Python & Poetry Environment
-         uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+         uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
          with:
+           poetry-version: 2.0.1
            python-version: ${{ matrix.python-version }}
        - name: Run security linter
          run: poetry run nox -s lint:security
@@ -144,8 +151,9 @@ jobs:
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
         with:
+          poetry-version: 2.0.1
           python-version: ${{ matrix.python-version }}
 
       - name: Calculate Test Coverage

.github/workflows/gh-pages.yml

@@ -16,7 +16,9 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Build Documentation
         run: |

.github/workflows/merge-gate.yml

@@ -12,16 +12,52 @@ jobs:
     name: Fast
     uses: ./.github/workflows/checks.yml
 
-  slow-checks:
-    name: SaaS Tests
+  slow-checks-approval:
+    name: Slow Check Approval
     runs-on: ubuntu-24.04
     environment: manual-approval
-
     # Replace the steps below with the required actions 
     # and/or add additional jobs if required
     # Note: 
     #   If you add additional jobs, make sure they are added as a requirement 
     #   to the approve-merge job's input requirements (needs).
+    steps:
+      - run: echo "Manual Approval"
+
+  saas-tests:
+    name: SaaS Tests
+    runs-on: ubuntu-24.04
+    needs: [ slow-checks-approval ]
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
+
+      - name: Allow unprivileged user namespaces
+        run: |
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0      
+
+      - name: Tests
+        env:
+          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
+          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
+          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
+        run: poetry run pytest -rA --setup-show --backend=saas test/integration/test_cloud_storage.py
+
+  large-runner-tests:
+    name: Text AI Tests
+    runs-on:
+      labels: int-linux-x64-4core-ubuntu24.04-1
+    environment: text-ai-prerelease
+    needs: [ slow-checks-approval ]
+
     steps:
       - name: SCM Checkout
         uses: actions/checkout@v4
@@ -44,25 +80,26 @@ jobs:
           sudo rm -rf /opt/ghc
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Allow unprivileged user namespaces
         run: |
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0      
 
       - name: Tests
         env:
-          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
-          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
-          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
-        run: poetry run pytest --backend=saas test/integration
+          TXAIE_PRE_RELEASE_URL: ${{ vars.ZIP_URL }}
+          TXAIE_PRE_RELEASE_PASSWORD: ${{ secrets.ZIP_PASSWORD }}
+        run: poetry run pytest -rA --setup-show test/integration/test_text_ai_extension_wrapper.py
 
   # This job ensures inputs have been executed successfully.
   approve-merge:
     name: Allow Merge 
     runs-on: ubuntu-latest
     # If you need additional jobs to be part of the merge gate, add them below
-    needs: [ fast-checks, slow-checks ]
+    needs: [ fast-checks, saas-tests, large-runner-tests ]
 
     # Each job requires a step, so we added this dummy step.
     steps:

.github/workflows/report.yml

@@ -21,7 +21,9 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0
+        with:
+          poetry-version: 2.0.1
 
       - name: Download Artifacts
         uses: actions/download-artifact@v4.1.8

doc/changes/unreleased.md

@@ -7,6 +7,8 @@
 * #146: Add interface for text_ai_extension_wrapper
 * #169: Added a function that creates a bucket-fs PathLike object
 * #173: Added an option to upload an SLC from file.
+* #167: Added partial implementation for text_ai_extension_wrapper
+* #168: Installed Span-UDFs in transformers_extension_wrapper 
 * #175: Added function to download and decrypt Text-AI pre-release.
 
 ## Refactoring

exasol/nb_connector/ai_lab_config.py

@@ -35,6 +35,10 @@ class AILabConfig(Enum):
     te_models_bfs_dir = auto()
     te_hf_connection = auto()
     te_models_cache_dir = auto()
+    txaie_bfs_connection = auto()
+    txaie_models_bfs_dir = auto()
+    txaie_models_cache_dir = auto()
+    txaie_slc_file_local_path = auto()
     sme_aws_bucket = auto()
     sme_aws_role = auto()
     sme_aws_connection = auto()

exasol/nb_connector/text_ai_extension_wrapper.py

@@ -1,3 +1,7 @@
+from os import PathLike
+
+from exasol.nb_connector.extension_wrapper_common import deploy_language_container, encapsulate_bucketfs_credentials
+from exasol.nb_connector.language_container_activation import ACTIVATION_KEY_PREFIX
 from typing import Optional, Generator
 from contextlib import contextmanager
 from pathlib import Path
@@ -8,10 +12,45 @@
 from exasol.nb_connector.secret_store import Secrets
 from exasol.nb_connector.ai_lab_config import AILabConfig as CKey
 
+# Models will be uploaded into directory BFS_MODELS_DIR in BucketFS.
+#
+# Models downloaded from the Huggingface archive to a local drive will be
+# cached in directory MODELS_CACHE_DIR.
+#
+# TXAIE uses the same directories as TE (see function initialize_te_extension)
+# as both extensions are using Huggingface Models. This also avoids confusion,
+# and ensures backwards compatibility.
+from exasol.nb_connector.transformers_extension_wrapper import BFS_MODELS_DIR, MODELS_CACHE_DIR
+
+
+PATH_IN_BUCKET = "TXAIE"
+""" Location in BucketFS bucket to upload data for TXAIE, e.g. its language container. """
+
 LANGUAGE_ALIAS = "PYTHON3_TXAIE"
 
 LATEST_KNOWN_VERSION = "???"
 
+ACTIVATION_KEY = ACTIVATION_KEY_PREFIX + "txaie"
+"""
+Activation SQL for the Text AI Extension will be saved in the secret store
+with this key.
+
+TXAIE brings its own Script Language Container (SLC) which needs to be
+activated by a dedicated SQL statement `ALTER SESSION SET SCRIPT_LANGUAGES`.  Applications
+can store the language definition in the configuration store (SCS) from the Notebook
+Connector's class `exasol.nb_connector.secret_store.Secrets`.
+
+Using `ACTIVATION_KEY` as defined key, TXAIE can provide convenient interfaces
+accepting only the SCS and retrieving all further data from the there.
+"""
+
+BFS_CONNECTION_PREFIX = "TXAIE_BFS"
+"""
+Prefix for Exasol CONNECTION objects containing a BucketFS location and
+credentials.
+"""
+
+
 
 @contextmanager
 def download_pre_release(conf: Secrets) -> Generator[tuple[Path, Path], None, None]:
@@ -49,6 +88,7 @@ def download_pre_release(conf: Secrets) -> Generator[tuple[Path, Path], None, No
             # Find and return the project wheel and the SLC
             project_wheel = next(tmp_path.glob("*.whl"))
             slc_tar_gz = next(tmp_path.glob("*.tar.gz"))
+            conf.save(CKey.txaie_slc_file_local_path, str(slc_tar_gz))
             yield project_wheel, slc_tar_gz
 
 
@@ -67,16 +107,18 @@ def deploy_licence(conf: Secrets,
             Optional. Content of a licence given as a string.
 
     """
-    pass
+    raise NotImplementedError('Currently this is not implemented, '
+                              'will be changed once the licensing process is finalized.')
+
 
 
 def initialize_text_ai_extension(conf: Secrets,
                                  container_file: Optional[Path] = None,
-                                 version: Optional[str] = LATEST_KNOWN_VERSION,
+                                 version: Optional[str] = None,
                                  language_alias: str = LANGUAGE_ALIAS,
                                  run_deploy_container: bool = True,
-                                 run_deploy_scripts: bool = True,
-                                 run_upload_models: bool = True,
+                                 run_deploy_scripts: bool = False,
+                                 run_upload_models: bool = False,
                                  run_encapsulate_bfs_credentials: bool = True,
                                  allow_override: bool = True) -> None:
     """
@@ -89,7 +131,8 @@ def initialize_text_ai_extension(conf: Secrets,
 
     If given a container_file path instead, installs the given container in the Bucketfs.
 
-    If neither is given, attempts to install the latest version from ???.
+    If neither is given, checks if txaie_slc_file_local_path is set and installs this SLC if found,
+    otherwise attempts to install the latest version from t.b.d.
 
     This function doesn't activate the language container. Instead, it gets the
     activation SQL using the same API and writes it to the secret store. The name
@@ -122,4 +165,63 @@ def initialize_text_ai_extension(conf: Secrets,
         allow_override:
             If True allows overriding the language definition.
     """
-    pass
+
+    # Create the name of the Exasol connection object
+    db_user = str(conf.get(CKey.db_user))
+    bfs_conn_name = "_".join([BFS_CONNECTION_PREFIX, db_user])
+    # As soon as the official release of TXAIE is available, the hard-coded value for
+    # container_name can be replaced by TXAIELanguageContainerDeployer.SLC_NAME,
+    # see https://github.com/exasol/notebook-connector/issues/179.
+    container_name = "exasol_text_ai_extension_container_release.tar.gz"
+
+
+    def from_ai_lab_config(key: CKey) -> Path | None:
+        entry = conf.get(key)
+        return Path(entry) if entry else None
+
+    if run_deploy_container:
+        if version:
+            install_text_ai_extension(version)
+            # Can run_upload_models, run_deploy_scripts,
+            # run_encapsulate_bfs_credentials, etc. be ignored here?
+            return
+
+        container_file = container_file or from_ai_lab_config(CKey.txaie_slc_file_local_path)
+        if not container_file:
+            install_text_ai_extension(LATEST_KNOWN_VERSION)
+        else:
+            deploy_language_container(
+                conf=conf,
+                path_in_bucket=PATH_IN_BUCKET,
+                language_alias=language_alias,
+                activation_key=ACTIVATION_KEY,
+                container_file=container_file,
+                container_name=container_name,
+                allow_override=allow_override,
+            )
+
+
+
+    if run_upload_models:
+        #  Install default Hugging Face models into the Bucketfs using
+        #  Transformers Extensions upload model functionality.
+        raise NotImplementedError('Implementation is waiting for TE release.')
+
+
+    if run_deploy_scripts:
+        raise NotImplementedError('Currently there are no Text-AI specific scripts to deploy.')
+
+
+    if run_encapsulate_bfs_credentials:
+        encapsulate_bucketfs_credentials(
+            conf, path_in_bucket=PATH_IN_BUCKET, connection_name=bfs_conn_name
+        )
+
+    # Update secret store
+    conf.save(CKey.txaie_bfs_connection, bfs_conn_name)
+    conf.save(CKey.txaie_models_bfs_dir, BFS_MODELS_DIR)
+    conf.save(CKey.txaie_models_cache_dir, MODELS_CACHE_DIR)
+
+
+def install_text_ai_extension(version: str) -> None:
+    raise NotImplementedError('Implementation is waiting for decision on where the releases will be hosted.')

exasol/nb_connector/transformers_extension_wrapper.py

@@ -63,7 +63,7 @@ def deploy_scripts(conf: Secrets,
         activation_sql = get_activation_sql(conf)
         conn.execute(activation_sql)
 
-        scripts_deployer = ScriptsDeployer(language_alias, conf.get(CKey.db_schema), conn)
+        scripts_deployer = ScriptsDeployer(language_alias, conf.get(CKey.db_schema), conn, install_all_scripts=True)
         scripts_deployer.deploy_scripts()