Merge remote-tracking branch 'origin/main'

Author: ahsimb

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
         run: true
 
   slow-test-detection:
-    name: Run Slow or Expensive Tests (e.g. SaaS)?
+    name: Run Integration Tests?
     runs-on: ubuntu-latest
     steps:
       - name: Detect Slow Tests
@@ -54,7 +54,7 @@ jobs:
       slow-tests
 
   run-slow-tests:
-    name: Run Slow or Expensive Tests (e.g. SaaS) if Requested
+    name: Run Integration Tests If Requested
     uses: ./.github/workflows/run-tests.yml
     needs: [ slow-test-detection ]
     secrets: inherit

.github/workflows/run-tests.yml

@@ -20,11 +20,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Set pytest markers
-        id: pytest-markers
-        if: ${{ ! inputs.slow-tests }}
-        run: echo slow-tests='-m "not saas"' >> "$GITHUB_OUTPUT"
-
       - name: SCM Checkout
         uses: actions/checkout@v4
 
@@ -33,20 +28,16 @@ jobs:
         with:
           python-version: ${{ inputs.python-version }}
 
-      - name: Run Tests and Calculate Coverage
+      - name: Run Unit Tests
+        if: ${{ ! inputs.slow-tests }}
+        run:
+          poetry run nox -s unit-tests
+
+      - name: Run Integration Tests
+        if: ${{ inputs.slow-tests }}
         env:
           SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
           SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
           SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
-          PYTEST_ADDOPTS: '${{ steps.pytest-markers.outputs.slow-tests }}'
-        run: |
-          echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS"
-          export PROJECT_SHORT_TAG=$(poetry run nox -s get-project-short-tag)
-          poetry run nox -s coverage -- -- --db-version ${{ inputs.exasol-version}}
-
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: .coverage
-          path: .coverage
-          overwrite: true
+        run:
+          poetry run nox -s integration-tests -- -- --db-version ${{ inputs.exasol-version}} --backend all

doc/changes/changes_0.5.0.md

@@ -7,13 +7,15 @@ Validation is implemented by
 * running a UDF inside it,
 * and making the UDF check for a specific file having been extracted from the SLC archive to be available on each node of the databasecluster.
 
-Addtionally this release refactors the existing CLI tests for the `LanguageContainerDeployer` which were integration tests involving the whole chain from the CLI down to the API, starting a database and uploading and activating SLCs.
+Additionally, this release refactors the existing CLI tests for the `LanguageContainerDeployer` which were integration tests involving the whole chain from the CLI down to the API, starting a database and uploading and activating SLCs.
 
 The existing integration tests have been split into
-* either unit tests just verifiying that the CLI options are passed to the API
+* either unit tests just verifying that the CLI options are passed to the API
 * or ordinary integration tests not using the CLI.
 
-This enables faster and more robust tests for the pure CLI-related features, faster turnaounds during development, and separation of concerns.
+This enables faster and more robust tests for the pure CLI-related features, faster turnarounds during development, and separation of concerns.
+
+The integration tests now use the pytest plugin `pytest-exasol-backend`.
 
 ## Features
 
@@ -23,7 +25,11 @@ This enables faster and more robust tests for the pure CLI-related features, fas
 # Refactoring
 
 * #51: Split CLI integration tests
+* #63: Removed the language_alias parameter from the LanguageContainerBuilder.
 
 ## Bug Fixing
 
-* #60: Fix handling pip requirements when creating an SLC
+* #60: Fixed handling pip requirements when creating an SLC.
+* #58: Fixed the bug in language_container_builder.find_path_backwards.
+* #36: Added SAAS_HOST environment variable to the user guide.
+* #35: Restoring pre-existing schema in the temp_schema module.

doc/user_guide/user-guide.md

@@ -44,7 +44,7 @@ another source.
 | bucketfs-password            |   [x]   |      | Env. [BUCKETFS_PASSWORD]                          |
 | bucketfs-use-https           |   [x]   |      | Optional boolean, defaults to False               |
 | bucket                       |   [x]   |      |                                                   |
-| saas-url                     |         | [x]  |                                                   |
+| saas-url                     |         | [x]  | Env. [SAAS_HOST]                                  |
 | saas-account-id              |         | [x]  | Env. [SAAS_ACCOUNT_ID]                            |
 | saas-database-id             |         | [x]  | Optional, Env. [SAAS_DATABASE_ID]                 |
 | saas-database-name           |         | [x]  | Optional, provide if the database_id is unknown   |

exasol/python_extension_common/deployment/language_container/flavor_base/build_steps.py

@@ -1,5 +1,4 @@
-# pylint: disable=import-error
-from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask    # type: ignore
+from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask    # type: ignore
 
 
 class AnalyzeDependencies(DockerFlavorAnalyzeImageTask):

exasol/python_extension_common/deployment/language_container/flavor_base/language_definition

@@ -6,10 +6,9 @@
 from pathlib import Path
 from importlib import resources
 
-# pylint: disable=import-error
 from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo   # type: ignore
-from exasol_script_languages_container_tool.lib import api            # type: ignore
-from exasol_script_languages_container_tool.lib.tasks.export.export_containers import ExportContainerResult     # type: ignore
+from exasol.slc import api            # type: ignore
+from exasol.slc.models.export_container_result import ExportContainerResult     # type: ignore
 
 
 def exclude_cuda(line: str) -> bool:
@@ -24,7 +23,8 @@ def find_path_backwards(target_path: str | Path, start_path: str | Path) -> Path
     error if the search is unsuccessful.
     """
     current_path = Path(start_path).parent
-    while current_path != current_path.root:
+    root = Path(current_path.root)
+    while current_path != root:
         result_path = Path(current_path, target_path)
         if result_path.exists():
             return result_path
@@ -44,9 +44,8 @@ def copy_slc_flavor(dest_dir: str | Path) -> None:
 
 class LanguageContainerBuilder:
 
-    def __init__(self, container_name: str, language_alias: str):
+    def __init__(self, container_name: str):
         self.container_name = container_name
-        self.language_alias = language_alias
         self._root_path: Path | None = None
         self._output_path: Path | None = None
 
@@ -58,9 +57,6 @@ def __enter__(self):
 
         # Copy the flavor into the working directory
         copy_slc_flavor(self.flavor_path)
-
-        # Write the language alias to the language definition
-        self._set_language_alias()
         return self
 
     def __exit__(self, *exc_details):
@@ -140,16 +136,6 @@ def export(self, export_path: str | Path | None = None) -> ExportContainerResult
                                    export_path=str(export_path))
         return export_result
 
-    def _set_language_alias(self) -> None:
-        """
-        Sets the language alias provided in the constractor to the language definition.
-        """
-        lang_def_path = self.flavor_base / 'language_definition'
-        lang_def_template = lang_def_path.read_text()
-        lang_def_text = lang_def_template.split("=", maxsplit=1)[1]
-        lang_def = f'{self.language_alias}={lang_def_text}'
-        lang_def_path.write_text(lang_def)
-
     def _add_requirements_to_flavor(self, project_directory: str | Path,
                                     requirement_filter: Callable[[str], bool] | None):
         """

exasol/python_extension_common/deployment/language_container_builder.py

@@ -109,6 +109,10 @@ def __init__(self,
             )
         logger.debug("Init %s", LanguageContainerDeployer.__name__)
 
+    @property
+    def pyexasol_connection(self) -> pyexasol.ExaConnection:
+        return self._pyexasol_conn
+
     def download_and_run(self, url: str,
                          bucket_file_path: str,
                          alter_system: bool = True,

exasol/python_extension_common/deployment/language_container_deployer.py

@@ -7,8 +7,13 @@
 from tenacity import retry
 from tenacity.stop import stop_after_attempt
 
+
 @retry(reraise=True, stop=stop_after_attempt(3))
 def _create_random_schema(conn: pyexasol.ExaConnection, schema_name_length: int) -> str:
+    """
+    The function creates a schema with randomly generated name. It makes a few retries,
+    as it's theoretically possible to create a collision with an existing schema.
+    """
 
     schema = ''.join(random.choice(string.ascii_letters)
                      for _ in range(schema_name_length))
@@ -17,8 +22,20 @@ def _create_random_schema(conn: pyexasol.ExaConnection, schema_name_length: int)
     return schema
 
 
+def _get_schema(conn: pyexasol.ExaConnection) -> str | None:
+    return conn.execute(f"SELECT CURRENT_SCHEMA;").fetchval()
+
+
+def _set_schema(conn: pyexasol.ExaConnection, schema: str | None):
+    if schema:
+        conn.execute(f'OPEN SCHEMA "{schema}";')
+    else:
+        conn.execute("CLOSE SCHEMA;")
+
+
 def _delete_schema(conn: pyexasol.ExaConnection, schema: str) -> None:
     sql = f'DROP SCHEMA IF EXISTS "{schema}" CASCADE;'
+    conn.execute(query=sql)
 
 
 @contextmanager
@@ -33,9 +50,11 @@ def temp_schema(conn: pyexasol.ExaConnection,
     conn                - pyexasol connection.
     schema_name_length  - Number of characters in the temporary schema name.
     """
+    current_schema = _get_schema(conn)
     schema = ''
     try:
         schema = _create_random_schema(conn, schema_name_length)
         yield schema
     finally:
         _delete_schema(conn, schema)
+        _set_schema(conn, current_schema)