diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index 51a5d1456..d987066ec 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -5,8 +5,6 @@ on: paths: - ".github/workflows/kubernetes_test.yaml" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -20,8 +18,6 @@ on: paths: - ".github/workflows/kubernetes_test.yaml" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -150,14 +146,14 @@ jobs: env: CYPRESS_BASE_URL: https://github-actions.nebari.dev/ with: - working-directory: tests_e2e + working-directory: tests/tests_e2e - name: Playwright Tests env: KEYCLOAK_USERNAME: ${{ env.CYPRESS_EXAMPLE_USER_NAME }} KEYCLOAK_PASSWORD: ${{ env.CYPRESS_EXAMPLE_USER_PASSWORD }} NEBARI_FULL_URL: https://github-actions.nebari.dev/ - working-directory: tests_e2e/playwright + working-directory: tests/tests_e2e/playwright run: | # create environment file envsubst < .env.tpl > .env @@ -170,15 +166,15 @@ jobs: with: name: e2e-cypress path: | - ./tests_e2e/cypress/screenshots/ - ./tests_e2e/cypress/videos/ - ./tests_e2e/playwright/videos/ + ./tests/tests_e2e/cypress/screenshots/ + ./tests/tests_e2e/cypress/videos/ + ./tests/tests_e2e/playwright/videos/ - name: Deployment Pytests run: | export KEYCLOAK_USERNAME=${CYPRESS_EXAMPLE_USER_NAME} export KEYCLOAK_PASSWORD=${CYPRESS_EXAMPLE_USER_PASSWORD} - pytest tests_deployment/ -v -s + pytest tests/tests_deployment/ -v -s - name: JupyterHub Notebook Tests # run jhub-client after pytest since jhubctl can cleanup @@ -192,7 +188,7 @@ jobs: --validate --no-verify-ssl \ --kernel python3 \ --stop-server \ - --notebook tests_deployment/assets/notebook/simple.ipynb \ + --notebook tests/tests_deployment/assets/notebook/simple.ipynb \ ### CLEANUP AFTER TESTS - name: Cleanup nebari deployment diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index 9bd764ebb..f56717abd 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -10,8 +10,6 @@ on: - ".github/failed-workflow-issue-templates/test-provider.md" - ".github/actions/publish-from-template" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -23,8 +21,6 @@ on: paths: - ".github/workflows/test-provider.yaml" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/**" - "scripts/**" - "src/**" - "pyproject.toml" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd30590be..522666870 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,8 +5,6 @@ on: paths: - ".github/workflows/test.yaml" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/cypress/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -19,8 +17,6 @@ on: paths: - ".github/workflows/test.yaml" - "tests/**" - - "tests_deployment/**" - - "tests_e2e/cypress/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -56,4 +52,4 @@ jobs: - name: Test Nebari run: | pytest --version - pytest --ignore=tests_deployment --ignore=tests_e2e/playwright + pytest --ignore=tests/tests_deployment --ignore=tests/tests_e2e/playwright --ignore=tests/tests_integration diff --git a/.github/workflows/test_integration.yaml b/.github/workflows/test_integration.yaml index 5607649ea..00253d067 100644 --- a/.github/workflows/test_integration.yaml +++ b/.github/workflows/test_integration.yaml @@ -1,4 +1,4 @@ -name: "Deploy on Digital Ocean" +name: "Integration Tests" on: schedule: @@ -12,6 +12,13 @@ jobs: permissions: id-token: write contents: read + strategy: + matrix: + provider: + - aws + - do + - gcp + fail-fast: false steps: - name: "Checkout Infrastructure" uses: actions/checkout@v3 @@ -21,6 +28,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.11 + - name: Install Nebari + run: | + pip install .[dev] + conda install --quiet --yes conda-build + playwright install - name: Retrieve secret from Vault uses: hashicorp/vault-action@v2.5.0 @@ -34,19 +46,46 @@ jobs: kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci project_id | PROJECT_ID; kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci workload_identity_provider | GCP_WORKFLOW_PROVIDER; kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci service_account_name | GCP_SERVICE_ACCOUNT; + kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID; + kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID; kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN; kv/data/repository/nebari-dev/nebari/cloudflare/internal-devops@quansight.com/nebari-dev-ci token | CLOUDFLARE_TOKEN; - - name: Install Nebari + - name: 'Authenticate to GCP' + if: ${{ matrix.provider == 'gcp' }} + uses: 'google-github-actions/auth@v1' + with: + token_format: access_token + create_credentials_file: 'true' + workload_identity_provider: ${{ env.GCP_WORKFLOW_PROVIDER }} + service_account: ${{ env.GCP_SERVICE_ACCOUNT }} + + - name: Set required environment variables + if: ${{ matrix.provider == 'gcp' }} run: | - pip install .[dev] - conda install --quiet --yes conda-build + echo "GOOGLE_CREDENTIALS=${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" >> $GITHUB_ENV + + - name: Authenticate to AWS + if: ${{ matrix.provider == 'aws' }} + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + role-session-name: github-action + aws-region: us-west-2 + + - name: Set Environment AWS + if: ${{ matrix.provider == 'aws' }} + run: | + echo "AWS_REGION=us-west-2" >> $GITHUB_ENV + + - name: Set Environment DO + if: ${{ matrix.provider == 'do' }} + run: | + echo "SPACES_ACCESS_KEY_ID=${{ secrets.SPACES_ACCESS_KEY_ID }}" >> $GITHUB_ENV + echo "SPACES_SECRET_ACCESS_KEY=${{ secrets.SPACES_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV + echo "NEBARI_K8S_VERSION"=1.25.12-do.0 >> $GITHUB_ENV - name: Integration Tests run: | pytest --version - pytest tests_integration/ -vvv -s - env: - NEBARI_K8S_VERSION: 1.25.12-do.0 - SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }} - SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }} + pytest tests/tests_integration/ -vvv -s -m ${{ matrix.provider }} diff --git a/pytest.ini b/pytest.ini index 5d6d3d52b..89f5ec586 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,16 @@ addopts = -Werror markers = conda: conda required to run this test (deselect with '-m \"not conda\"') + aws: deploy on aws + do: deploy on do + gcp: deploy on gcp + azure: deploy on azure + gpu: test gpu working properly testpaths = tests xfail_strict = True + +log_format = %(asctime)s %(levelname)9s %(lineno)4s %(module)s: %(message)s +log_date_format = %Y-%m-%d %H:%M:%S +log_cli = True +log_cli_level = INFO diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 49a53b9bb..4579298c7 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -62,8 +62,9 @@ def create_user( rich.print( f"Creating user=[green]{username}[/green] without password (none supplied)" ) - keycloak_admin.create_user(payload) + user = keycloak_admin.create_user(payload) rich.print(f"Created user=[green]{username}[/green]") + return user def list_users(keycloak_admin: keycloak.KeycloakAdmin): diff --git a/tests_deployment/__init__.py b/tests/common/__init__.py similarity index 100% rename from tests_deployment/__init__.py rename to tests/common/__init__.py diff --git a/tests/common/config_mod_utils.py b/tests/common/config_mod_utils.py new file mode 100644 index 000000000..dce0b4fde --- /dev/null +++ b/tests/common/config_mod_utils.py @@ -0,0 +1,117 @@ +import dataclasses +import typing + +PREEMPTIBLE_NODE_GROUP_NAME = "preemptible-node-group" + + +@dataclasses.dataclass +class GPUConfig: + cloud: str + gpu_name: str + node_selector: str + node_selector_val: str + extra_config: dict + min_nodes: typing.Optional[int] = 0 + max_nodes: typing.Optional[int] = 2 + node_group_name: typing.Optional[str] = "gpu-node" + docker_image: typing.Optional[str] = "quay.io/nebari/nebari-jupyterlab-gpu:2023.7.1" + + def node(self): + return { + "instance": self.gpu_name, + "min_nodes": self.min_nodes, + "max_nodes": self.max_nodes, + **self.extra_config, + } + + +AWS_GPU_CONF = GPUConfig( + cloud="amazon_web_services", + gpu_name="g4dn.xlarge", + node_selector="beta.kubernetes.io/instance-type", + node_selector_val="g4dn.xlarge", + extra_config={ + "single_subnet": False, + "gpu": True, + }, +) + + +GCP_GPU_CONF = GPUConfig( + cloud="google_cloud_platform", + gpu_name="n1-standard-16", + node_selector="cloud.google.com/gke-nodepool", + node_selector_val="g4dn.xlarge", + extra_config={"guest_accelerators": [{"name": "nvidia-tesla-t4", "count": 1}]}, +) + + +GPU_CONFIG = { + "aws": AWS_GPU_CONF, + "gcp": GCP_GPU_CONF, +} + + +def _create_gpu_environment(): + return { + "name": "gpu", + "channels": ["pytorch", "nvidia", "conda-forge"], + "dependencies": [ + "python=3.10.8", + "ipykernel=6.21.0", + "ipywidgets==7.7.1", + "torchvision", + "torchaudio", + "cudatoolkit", + "pytorch-cuda=11.7", + "pytorch::pytorch", + ], + } + + +def add_gpu_config(config, cloud="aws"): + gpu_config = GPU_CONFIG.get(cloud) + if not gpu_config: + raise ValueError(f"GPU not supported/tested on {cloud}") + + gpu_node = gpu_config.node() + gpu_docker_image = gpu_config.docker_image + jupyterlab_profile = { + "display_name": "GPU Instance", + "description": "4 CPU / 16GB RAM / 1 NVIDIA T4 GPU (16 GB GPU RAM)", + "groups": ["gpu-access"], + "kubespawner_override": { + "image": gpu_docker_image, + "cpu_limit": 4, + "cpu_guarantee": 3, + "mem_limit": "16G", + "mem_guarantee": "10G", + "extra_resource_limits": {"nvidia.com/gpu": 1}, + "node_selector": { + gpu_config.node_selector: gpu_config.node_selector_val, + }, + }, + } + config[gpu_config.cloud]["node_groups"][gpu_config.node_group_name] = gpu_node + config["profiles"]["jupyterlab"].append(jupyterlab_profile) + config["environments"]["environment-gpu.yaml"] = _create_gpu_environment() + return config + + +def add_preemptible_node_group(config, cloud="aws"): + if cloud == "aws": + cloud_name = "amazon_web_services" + instance_name = "m5.xlarge" + elif cloud == "gcp": + cloud_name = "google_cloud_platform" + instance_name = "n1-standard-8" + else: + raise ValueError("Invalid cloud for preemptible config") + config[cloud_name]["node_groups"][PREEMPTIBLE_NODE_GROUP_NAME] = { + "instance": instance_name, + "min_nodes": 1, + "max_nodes": 5, + "single_subnet": False, + "preemptible": True, + } + return config diff --git a/tests_e2e/playwright/navigator.py b/tests/common/navigator.py similarity index 97% rename from tests_e2e/playwright/navigator.py rename to tests/common/navigator.py index 5bf0d3efe..b906822dc 100644 --- a/tests_e2e/playwright/navigator.py +++ b/tests/common/navigator.py @@ -189,7 +189,7 @@ def start_server(self) -> None: """ # wait for the page to load logout_button = self.page.get_by_text("Logout", exact=True) - logout_button.wait_for(state="attached") + logout_button.wait_for(state="attached", timeout=90000) # if server is not yet running start_locator = self.page.get_by_role("button", name="Start My Server") @@ -214,6 +214,7 @@ def start_server(self) -> None: self.page.wait_for_url( urllib.parse.urljoin(self.nebari_url, f"user/{self.username}/*"), wait_until="networkidle", + timeout=180000, ) # the jupyter page loads independent of network activity so here @@ -235,8 +236,8 @@ def _check_for_kernel_popup(self): True if the kernel popup is open. """ self.page.wait_for_load_state("networkidle") + time.sleep(3) visible = self.page.get_by_text("Select Kernel", exact=True).is_visible() - return visible def reset_workspace(self): @@ -247,7 +248,7 @@ def reset_workspace(self): * reset file browser is reset to root * Finally, ensure that the Launcher screen is showing """ - logger.debug(">>> Reset JupyterLab workspace") + logger.info(">>> Reset JupyterLab workspace") # server is already running and there is no popup popup = self._check_for_kernel_popup() @@ -307,7 +308,11 @@ def _set_environment_via_popup(self, kernel=None): if kernel is None: # close dialog (deal with the two formats of this dialog) try: - self.page.get_by_text("Cancel", exact=True).click() + cancel_button = self.page.get_by_text("Cancel", exact=True) + if cancel_button.is_visible(): + cancel_button.click() + else: + self.page.mouse.click(0, 0) except Exception: self.page.locator("div").filter(has_text="No KernelSelect").get_by_role( "button", name="No Kernel" diff --git a/tests_e2e/playwright/test_data/test_notebook_output.ipynb b/tests/common/notebooks/test_notebook_output.ipynb similarity index 100% rename from tests_e2e/playwright/test_data/test_notebook_output.ipynb rename to tests/common/notebooks/test_notebook_output.ipynb diff --git a/tests_e2e/playwright/conftest.py b/tests/common/playwright_fixtures.py similarity index 58% rename from tests_e2e/playwright/conftest.py rename to tests/common/playwright_fixtures.py index bb629917f..388f6ef4b 100644 --- a/tests_e2e/playwright/conftest.py +++ b/tests/common/playwright_fixtures.py @@ -4,13 +4,14 @@ import dotenv import pytest -from navigator import Navigator + +from tests.common.navigator import Navigator logger = logging.getLogger() @pytest.fixture(scope="session") -def _navigator_session(browser_name, pytestconfig): +def _navigator_session(request, browser_name, pytestconfig): """Set up a navigator instance, login with username/password, start a server. Teardown when session is complete. Do not use this for individual tests, use `navigator` fixture @@ -21,14 +22,18 @@ def _navigator_session(browser_name, pytestconfig): # the error. try: nav = Navigator( - nebari_url=os.environ["NEBARI_FULL_URL"], - username=os.environ["KEYCLOAK_USERNAME"], - password=os.environ["KEYCLOAK_PASSWORD"], + nebari_url=request.param.get("nebari_url") or os.environ["NEBARI_FULL_URL"], + username=request.param.get("keycloak_username") + or os.environ["KEYCLOAK_USERNAME"], + password=request.param.get("keycloak_password") + or os.environ["KEYCLOAK_PASSWORD"], headless=not pytestconfig.getoption("--headed"), slow_mo=pytestconfig.getoption("--slowmo"), browser=browser_name, auth="password", - instance_name="small-instance", # small-instance included by default + instance_name=request.param.get( + "instance_name" + ), # small-instance included by default video_dir="videos/", ) except Exception as e: @@ -56,4 +61,16 @@ def navigator(_navigator_session): @pytest.fixture(scope="session") def test_data_root(): here = Path(__file__).parent - return here / "test_data" + return here / "notebooks" + + +def navigator_parameterized( + nebari_url=None, keycloak_username=None, keycloak_password=None, instance_name=None +): + param = { + "instance_name": instance_name, + "nebari_url": nebari_url, + "keycloak_username": keycloak_username, + "keycloak_password": keycloak_password, + } + return pytest.mark.parametrize("_navigator_session", [param], indirect=True) diff --git a/tests/common/run_notebook.py b/tests/common/run_notebook.py new file mode 100644 index 000000000..e55b874d1 --- /dev/null +++ b/tests/common/run_notebook.py @@ -0,0 +1,173 @@ +import logging +import re +import time +from pathlib import Path + +from tests.common.navigator import Navigator + +logger = logging.getLogger() + + +class Notebook: + def __init__(self, navigator: Navigator): + self.nav = navigator + self.nav.initialize + + def run( + self, + path, + expected_outputs, + conda_env, + runtime=30000, + retry=2, + exact_match=True, + ): + """Run jupyter notebook and check for expected output text anywhere on + the page. + + Note: This will look for and exact match of expected_output_text + _anywhere_ on the page so be sure that your text is unique. + + Conda environments may still be being built shortly after deployment. + + conda_env: str + Name of conda environment. Python conda environments have the + structure "conda-env-nebari-git-nebari-git-dashboard-py" where + the actual name of the environment is "dashboard". + """ + logger.debug(f">>> Running notebook: {path}") + filename = Path(path).name + + # navigate to specific notebook + self.open_notebook(path) + # make sure the focus is on the dashboard tab we want to run + self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() + self.nav.set_environment(kernel=conda_env) + + # make sure that this notebook is one currently selected + self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() + + for i in range(retry): + self._restart_run_all() + # Wait for a couple of seconds to make sure it's re-started + time.sleep(2) + self._wait_for_commands_completion() + all_outputs = self._get_outputs() + self.assert_match_all_outputs(expected_outputs, all_outputs) + + def create_notebook(self, conda_env=None): + file_locator = self.nav.page.get_by_text("File", exact=True) + file_locator.wait_for( + timeout=self.nav.wait_for_server_spinup, + state="attached", + ) + file_locator.click() + submenu = self.nav.page.locator('[data-type="submenu"]').all() + submenu[0].click() + self.nav.page.get_by_role("menuitem", name="Notebook").get_by_text( + "Notebook", exact=True + ).click() + self.nav.page.wait_for_load_state("networkidle") + # make sure the focus is on the dashboard tab we want to run + # self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() + self.nav.set_environment(kernel=conda_env) + + def open_notebook(self, path): + file_locator = self.nav.page.get_by_text("File", exact=True) + file_locator.wait_for( + timeout=self.nav.wait_for_server_spinup, + state="attached", + ) + file_locator.click() + self.nav.page.get_by_role("menuitem", name="Open from Path…").get_by_text( + "Open from Path…" + ).click() + self.nav.page.get_by_placeholder("/path/relative/to/jlab/root").fill(path) + self.nav.page.get_by_role("button", name="Open", exact=True).click() + # give the page a second to open, otherwise the options in the kernel + # menu will be disabled. + self.nav.page.wait_for_load_state("networkidle") + + if self.nav.page.get_by_text( + "Could not find path:", + exact=False, + ).is_visible(): + logger.debug("Path to notebook is invalid") + raise RuntimeError("Path to notebook is invalid") + + def assert_code_output(self, code, expected_output): + self.run_in_last_cell(code) + self._wait_for_commands_completion() + outputs = self._get_outputs() + self.assert_match_output(expected_output, outputs[-1]) + + def run_in_last_cell(self, code): + self._create_new_cell() + cell = self._get_last_cell() + cell.click() + cell.type(code) + # Wait for it to be ready to be executed + time.sleep(1) + cell.press("Shift+Enter") + # Wait for execution to start + time.sleep(0.5) + + def _create_new_cell(self): + new_cell_button = self.nav.page.query_selector( + 'button[data-command="notebook:insert-cell-below"]' + ) + new_cell_button.click() + + def _get_last_cell(self): + cells = self.nav.page.locator(".CodeMirror-code").all() + for cell in reversed(cells): + if cell.is_visible(): + return cell + raise ValueError("Unable to get last cell") + + def _wait_for_commands_completion(self, timeout=120): + elapsed_time = 0 + start_time = time.time() + still_visible = True + while elapsed_time < timeout: + running = self.nav.page.get_by_text("[*]").all() + still_visible = any(list(map(lambda r: r.is_visible(), running))) + elapsed_time = time.time() - start_time + time.sleep(1) + if not still_visible: + break + if still_visible: + raise ValueError( + f"Timeout Waited for commands to finish, " + f"but couldn't finish in {timeout} sec" + ) + + def _get_outputs(self): + output_elements = self.nav.page.query_selector_all(".jp-OutputArea-output") + text_content = [element.text_content().strip() for element in output_elements] + return text_content + + def assert_match_all_outputs(self, expected_outputs, actual_outputs): + for ex, act in zip(expected_outputs, actual_outputs): + self.assert_match_output(ex, act) + + def assert_match_output(self, expected_output, actual_output): + if isinstance(expected_output, re.Pattern): + assert re.match(expected_output, actual_output) + else: + assert expected_output == actual_output + + def _restart_run_all(self): + # restart run all cells + self.nav.page.get_by_text("Kernel", exact=True).click() + self.nav.page.get_by_role( + "menuitem", name="Restart Kernel and Run All Cells…" + ).get_by_text("Restart Kernel and Run All Cells…").click() + + # Restart dialog appears most, but not all of the time (e.g. set + # No Kernel, then Restart Run All) + restart_dialog_button = self.nav.page.get_by_role( + "button", name="Restart", exact=True + ) + if restart_dialog_button.is_visible(): + restart_dialog_button.click() diff --git a/tests/conftest.py b/tests/conftest.py index 65abddf33..9a98c74d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,67 +1 @@ -from unittest.mock import Mock - -import pytest - -from tests.utils import INIT_INPUTS, NEBARI_CONFIG_FN, PRESERVED_DIR - - -@pytest.fixture(params=INIT_INPUTS) -def setup_fixture(request, monkeypatch, tmp_path): - """This fixture helps simplify writing tests by: - - parametrizing the different cloud provider inputs in a single place - - creating a tmp directory (and file) for the `nebari-config.yaml` to be save to - - monkeypatching functions that call out to external APIs. - """ - render_config_inputs = request.param - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - def _mock_kubernetes_versions(grab_latest_version=False): - # template for all `kubernetes_versions` calls - # monkeypatched to avoid making outbound API calls in CI - k8s_versions = ["1.18", "1.19", "1.20"] - m = Mock() - m.return_value = k8s_versions - if grab_latest_version: - m.return_value = k8s_versions[-1] - return m - - if cloud_provider == "aws": - monkeypatch.setattr( - "_nebari.utils.amazon_web_services.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "azure": - monkeypatch.setattr( - "_nebari.utils.azure_cloud.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "do": - monkeypatch.setattr( - "_nebari.utils.digital_ocean.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "gcp": - monkeypatch.setattr( - "_nebari.utils.google_cloud.kubernetes_versions", - _mock_kubernetes_versions(), - ) - - output_directory = tmp_path / f"{cloud_provider}_output_dir" - output_directory.mkdir() - nebari_config_loc = output_directory / NEBARI_CONFIG_FN - - # data that should NOT be deleted when `nebari render` is called - # see test_render.py::test_remove_existing_renders - preserved_directory = output_directory / PRESERVED_DIR - preserved_directory.mkdir() - preserved_filename = preserved_directory / "file.txt" - preserved_filename.write_text("This is a test...") - - yield (nebari_config_loc, render_config_inputs) +pytest_plugins = ["tests.common.playwright_fixtures"] diff --git a/tests_integration/__init__.py b/tests/tests_deployment/__init__.py similarity index 100% rename from tests_integration/__init__.py rename to tests/tests_deployment/__init__.py diff --git a/tests_deployment/assets/notebook/simple.ipynb b/tests/tests_deployment/assets/notebook/simple.ipynb similarity index 100% rename from tests_deployment/assets/notebook/simple.ipynb rename to tests/tests_deployment/assets/notebook/simple.ipynb diff --git a/tests_deployment/constants.py b/tests/tests_deployment/constants.py similarity index 100% rename from tests_deployment/constants.py rename to tests/tests_deployment/constants.py diff --git a/tests_deployment/test_dask_gateway.py b/tests/tests_deployment/test_dask_gateway.py similarity index 91% rename from tests_deployment/test_dask_gateway.py rename to tests/tests_deployment/test_dask_gateway.py index 417056c43..78b02de88 100644 --- a/tests_deployment/test_dask_gateway.py +++ b/tests/tests_deployment/test_dask_gateway.py @@ -3,8 +3,8 @@ import dask_gateway import pytest -from tests_deployment import constants -from tests_deployment.utils import get_jupyterhub_token, monkeypatch_ssl_context +from tests.tests_deployment import constants +from tests.tests_deployment.utils import get_jupyterhub_token, monkeypatch_ssl_context monkeypatch_ssl_context() diff --git a/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py similarity index 97% rename from tests_deployment/test_jupyterhub_ssh.py rename to tests/tests_deployment/test_jupyterhub_ssh.py index b1c3691f5..8ecfa4f40 100644 --- a/tests_deployment/test_jupyterhub_ssh.py +++ b/tests/tests_deployment/test_jupyterhub_ssh.py @@ -4,8 +4,8 @@ import paramiko import pytest -from tests_deployment import constants -from tests_deployment.utils import ( +from tests.tests_deployment import constants +from tests.tests_deployment.utils import ( escape_string, get_jupyterhub_token, monkeypatch_ssl_context, diff --git a/tests_deployment/utils.py b/tests/tests_deployment/utils.py similarity index 97% rename from tests_deployment/utils.py rename to tests/tests_deployment/utils.py index dd28ff799..5be0f948d 100644 --- a/tests_deployment/utils.py +++ b/tests/tests_deployment/utils.py @@ -5,7 +5,7 @@ import escapism import requests -from tests_deployment import constants +from tests.tests_deployment import constants def get_jupyterhub_session(): diff --git a/tests_e2e/.gitignore b/tests/tests_e2e/.gitignore similarity index 100% rename from tests_e2e/.gitignore rename to tests/tests_e2e/.gitignore diff --git a/tests_e2e/cypress.json b/tests/tests_e2e/cypress.json similarity index 100% rename from tests_e2e/cypress.json rename to tests/tests_e2e/cypress.json diff --git a/tests_e2e/cypress/.gitignore b/tests/tests_e2e/cypress/.gitignore similarity index 100% rename from tests_e2e/cypress/.gitignore rename to tests/tests_e2e/cypress/.gitignore diff --git a/tests_e2e/cypress/integration/main.js b/tests/tests_e2e/cypress/integration/main.js similarity index 100% rename from tests_e2e/cypress/integration/main.js rename to tests/tests_e2e/cypress/integration/main.js diff --git a/tests_e2e/cypress/notebooks/BasicTest.ipynb b/tests/tests_e2e/cypress/notebooks/BasicTest.ipynb similarity index 100% rename from tests_e2e/cypress/notebooks/BasicTest.ipynb rename to tests/tests_e2e/cypress/notebooks/BasicTest.ipynb diff --git a/tests_e2e/cypress/plugins/index.js b/tests/tests_e2e/cypress/plugins/index.js similarity index 100% rename from tests_e2e/cypress/plugins/index.js rename to tests/tests_e2e/cypress/plugins/index.js diff --git a/tests_e2e/cypress/support/index.js b/tests/tests_e2e/cypress/support/index.js similarity index 100% rename from tests_e2e/cypress/support/index.js rename to tests/tests_e2e/cypress/support/index.js diff --git a/tests_e2e/package-lock.json b/tests/tests_e2e/package-lock.json similarity index 100% rename from tests_e2e/package-lock.json rename to tests/tests_e2e/package-lock.json diff --git a/tests_e2e/package.json b/tests/tests_e2e/package.json similarity index 100% rename from tests_e2e/package.json rename to tests/tests_e2e/package.json diff --git a/tests_e2e/playwright/.env.tpl b/tests/tests_e2e/playwright/.env.tpl similarity index 100% rename from tests_e2e/playwright/.env.tpl rename to tests/tests_e2e/playwright/.env.tpl diff --git a/tests_e2e/playwright/README.md b/tests/tests_e2e/playwright/README.md similarity index 100% rename from tests_e2e/playwright/README.md rename to tests/tests_e2e/playwright/README.md diff --git a/tests/tests_e2e/playwright/test_playwright.py b/tests/tests_e2e/playwright/test_playwright.py new file mode 100644 index 000000000..30035b8b5 --- /dev/null +++ b/tests/tests_e2e/playwright/test_playwright.py @@ -0,0 +1,18 @@ +from tests.common.playwright_fixtures import navigator_parameterized +from tests.common.run_notebook import Notebook + + +@navigator_parameterized(instance_name="small-instance") +def test_notebook(navigator, test_data_root): + test_app = Notebook(navigator=navigator) + notebook_name = "test_notebook_output.ipynb" + notebook_path = test_data_root / notebook_name + assert notebook_path.exists() + with open(notebook_path, "r") as notebook: + test_app.nav.write_file(filepath=notebook_name, content=notebook.read()) + test_app.run( + path=notebook_name, + expected_outputs=["success: 6"], + conda_env="conda-env-default-py", + runtime=60000, + ) diff --git a/tests_integration/README.md b/tests/tests_integration/README.md similarity index 100% rename from tests_integration/README.md rename to tests/tests_integration/README.md diff --git a/tests/notebooks/test-ipython-basic.ipynb b/tests/tests_integration/__init__.py similarity index 100% rename from tests/notebooks/test-ipython-basic.ipynb rename to tests/tests_integration/__init__.py diff --git a/tests/tests_integration/conftest.py b/tests/tests_integration/conftest.py new file mode 100644 index 000000000..7674a4b04 --- /dev/null +++ b/tests/tests_integration/conftest.py @@ -0,0 +1,4 @@ +pytest_plugins = [ + "tests.tests_integration.deployment_fixtures", + "tests.common.playwright_fixtures", +] diff --git a/tests/tests_integration/deployment_fixtures.py b/tests/tests_integration/deployment_fixtures.py new file mode 100644 index 000000000..4e5ad3301 --- /dev/null +++ b/tests/tests_integration/deployment_fixtures.py @@ -0,0 +1,171 @@ +import logging +import os +import random +import string +import uuid +import warnings +from pathlib import Path + +import pytest +from urllib3.exceptions import InsecureRequestWarning + +from _nebari.deploy import deploy_configuration +from _nebari.destroy import destroy_configuration +from _nebari.render import render_template +from _nebari.utils import yaml +from tests.common.config_mod_utils import add_gpu_config, add_preemptible_node_group +from tests.tests_unit.utils import render_config_partial + +DEPLOYMENT_DIR = "_test_deploy" + +logger = logging.getLogger(__name__) + + +def ignore_warnings(): + # Ignore this for now, as test is failing due to a + # DeprecationWarning and InsecureRequestWarning + warnings.filterwarnings("ignore", category=DeprecationWarning) + warnings.filterwarnings("ignore", category=InsecureRequestWarning) + + +@pytest.fixture(autouse=True) +def disable_warnings(): + ignore_warnings() + + +def _random_letters(length=5): + letters = string.ascii_letters + return "".join(random.choice(letters) for _ in range(length)).lower() + + +def _get_or_create_deployment_directory(cloud): + """This will create a directory to initialise and deploy + Nebari from. + """ + deployment_dirs = list(Path(Path(DEPLOYMENT_DIR) / cloud).glob(f"pytest{cloud}*")) + if deployment_dirs: + deployment_dir = deployment_dirs[0] + else: + project_name = f"pytest{cloud}{_random_letters()}" + deployment_dir = Path(Path(Path(DEPLOYMENT_DIR) / cloud) / project_name) + deployment_dir.mkdir(parents=True) + return deployment_dir + + +def _set_do_environment(): + os.environ["AWS_ACCESS_KEY_ID"] = os.environ["SPACES_ACCESS_KEY_ID"] + os.environ["AWS_SECRET_ACCESS_KEY"] = os.environ["SPACES_SECRET_ACCESS_KEY"] + + +def _set_nebari_creds_in_environment(config): + os.environ["NEBARI_FULL_URL"] = f"https://{config['domain']}/" + os.environ["KEYCLOAK_USERNAME"] = "pytest" + os.environ["KEYCLOAK_PASSWORD"] = os.environ.get( + "PYTEST_KEYCLOAK_PASSWORD", uuid.uuid4().hex + ) + + +def _create_nebari_user(config): + import keycloak + + from _nebari.keycloak import create_user, get_keycloak_admin_from_config + + keycloak_admin = get_keycloak_admin_from_config(config) + try: + user = create_user(keycloak_admin, "pytest", "pytest-password") + return user + except keycloak.KeycloakPostError as e: + if e.response_code == 409: + logger.info(f"User already exists: {e.response_body}") + + +@pytest.fixture(scope="session") +def deploy(request): + """Deploy Nebari on the given cloud, currently only DigitalOcean""" + ignore_warnings() + cloud = request.param + logger.info(f"Deploying: {cloud}") + if cloud == "do": + _set_do_environment() + deployment_dir = _get_or_create_deployment_directory(cloud) + config = render_config_partial( + project_name=deployment_dir.name, + namespace="dev", + nebari_domain=f"ci-{cloud}.nebari.dev", + cloud_provider=cloud, + ci_provider="github-actions", + auth_provider="github", + ) + # Generate certificate as well + config["certificate"] = { + "type": "lets-encrypt", + "acme_email": "internal-devops@quansight.com", + "acme_server": "https://acme-v02.api.letsencrypt.org/directory", + } + if cloud in ["aws", "gcp"]: + config = add_gpu_config(config, cloud=cloud) + config = add_preemptible_node_group(config, cloud=cloud) + + deployment_dir_abs = deployment_dir.absolute() + os.chdir(deployment_dir) + logger.info(f"Temporary directory: {deployment_dir}") + config_path = Path("nebari-config.yaml") + + if config_path.exists(): + with open(config_path) as f: + current_config = yaml.load(f) + + config["security"]["keycloak"]["initial_root_password"] = current_config[ + "security" + ]["keycloak"]["initial_root_password"] + + with open(config_path, "w") as f: + yaml.dump(config, f) + render_template(deployment_dir_abs, Path("nebari-config.yaml")) + failed = False + try: + logger.info("*" * 100) + logger.info(f"Deploying Nebari on {cloud}") + logger.info("*" * 100) + deploy_config = deploy_configuration( + config=config, + dns_provider="cloudflare", + dns_auto_provision=True, + disable_prompt=True, + disable_checks=False, + skip_remote_state_provision=False, + ) + _create_nebari_user(config) + _set_nebari_creds_in_environment(config) + yield deploy_config + except Exception as e: + failed = True + logger.error(f"Deploy Failed, Exception: {e}") + logger.exception(e) + logger.info("*" * 100) + logger.info("Tearing down") + _destroy(config) + if failed: + raise AssertionError("Deployment failed") + + +def _destroy(config): + try: + return destroy_configuration(config) + except Exception as e: + logger.exception(e) + logger.info("Destroy failed!") + raise + + +def on_cloud(param=None): + """Decorator to run tests on a particular cloud or all cloud.""" + clouds = ["aws", "do", "gcp"] + if param: + clouds = [param] if not isinstance(param, list) else param + + def _create_pytest_param(cloud): + return pytest.param(cloud, marks=getattr(pytest.mark, cloud)) + + all_clouds_param = map(_create_pytest_param, clouds) + return pytest.mark.parametrize("deploy", all_clouds_param, indirect=True) diff --git a/tests_integration/test_integration.py b/tests/tests_integration/test_all_clouds.py similarity index 86% rename from tests_integration/test_integration.py rename to tests/tests_integration/test_all_clouds.py index 7ea18dfc2..94be86df2 100644 --- a/tests_integration/test_integration.py +++ b/tests/tests_integration/test_all_clouds.py @@ -1,16 +1,10 @@ -import pytest import requests -from tests_integration.deployment_fixtures import ignore_warnings, on_cloud +from tests.tests_integration.deployment_fixtures import on_cloud -@pytest.fixture(autouse=True) -def disable_warnings(): - ignore_warnings() - - -@on_cloud("do") -def test_do_service_status(deploy): +@on_cloud() +def test_service_status(deploy): """Tests if deployment on DigitalOcean succeeds""" service_urls = deploy["stages/07-kubernetes-services"]["service_urls"]["value"] assert ( @@ -39,7 +33,7 @@ def test_do_service_status(deploy): ) -@on_cloud("do") +@on_cloud() def test_verify_keycloak_users(deploy): """Tests if keycloak is working and it has expected users""" keycloak_credentials = deploy["stages/05-kubernetes-keycloak"][ diff --git a/tests/tests_integration/test_gpu.py b/tests/tests_integration/test_gpu.py new file mode 100644 index 000000000..da78ea228 --- /dev/null +++ b/tests/tests_integration/test_gpu.py @@ -0,0 +1,26 @@ +import re + +import pytest + +from tests.common.playwright_fixtures import navigator_parameterized +from tests.common.run_notebook import Notebook +from tests.tests_integration.deployment_fixtures import on_cloud + + +@on_cloud(["aws", "gcp"]) +@pytest.mark.gpu +@navigator_parameterized(instance_name="gpu-instance") +def test_gpu(deploy, navigator, test_data_root): + test_app = Notebook(navigator=navigator) + conda_env = "gpu" + test_app.create_notebook( + conda_env=f"conda-env-nebari-git-nebari-git-{conda_env}-py" + ) + test_app.assert_code_output( + code="!nvidia-smi", + expected_output=re.compile(".*\n.*\n.*NVIDIA-SMI.*CUDA Version"), + ) + + test_app.assert_code_output( + code="import torch;torch.cuda.is_available()", expected_output="True" + ) diff --git a/tests/tests_integration/test_preemptible.py b/tests/tests_integration/test_preemptible.py new file mode 100644 index 000000000..084c3a1b7 --- /dev/null +++ b/tests/tests_integration/test_preemptible.py @@ -0,0 +1,34 @@ +import pytest +from kubernetes import client, config + +from tests.common.config_mod_utils import PREEMPTIBLE_NODE_GROUP_NAME +from tests.tests_integration.deployment_fixtures import on_cloud + + +@on_cloud() +def test_preemptible(request, deploy): + config.load_kube_config( + config_file=deploy["stages/02-infrastructure"]["kubeconfig_filename"]["value"] + ) + if request.node.get_closest_marker("aws"): + name_label = "eks.amazonaws.com/nodegroup" + preemptible_key = "eks.amazonaws.com/capacityType" + expected_value = "SPOT" + pytest.xfail("Preemptible instances are not supported on AWS atm") + + elif request.node.get_closest_marker("gcp"): + name_label = "cloud.google.com/gke-nodepool" + preemptible_key = "cloud.google.com/gke-preemptible" + expected_value = "true" + else: + pytest.skip("Unsupported cloud for preemptible") + raise ValueError("Invalid cloud for testing preemptible") + + api_instance = client.CoreV1Api() + nodes = api_instance.list_node() + node_labels_map = {} + for node in nodes.items: + node_name = node.metadata.labels[name_label] + node_labels_map[node_name] = node.metadata.labels + preemptible_node_group_labels = node_labels_map[PREEMPTIBLE_NODE_GROUP_NAME] + assert preemptible_node_group_labels.get(preemptible_key) == expected_value diff --git a/tests/tests_unit/__init__.py b/tests/tests_unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tests_unit/conftest.py b/tests/tests_unit/conftest.py new file mode 100644 index 000000000..ba588d1b1 --- /dev/null +++ b/tests/tests_unit/conftest.py @@ -0,0 +1,67 @@ +from unittest.mock import Mock + +import pytest + +from tests.tests_unit.utils import INIT_INPUTS, NEBARI_CONFIG_FN, PRESERVED_DIR + + +@pytest.fixture(params=INIT_INPUTS) +def setup_fixture(request, monkeypatch, tmp_path): + """This fixture helps simplify writing tests by: + - parametrizing the different cloud provider inputs in a single place + - creating a tmp directory (and file) for the `nebari-config.yaml` to be save to + - monkeypatching functions that call out to external APIs. + """ + render_config_inputs = request.param + ( + project, + namespace, + domain, + cloud_provider, + ci_provider, + auth_provider, + ) = render_config_inputs + + def _mock_kubernetes_versions(grab_latest_version=False): + # template for all `kubernetes_versions` calls + # monkeypatched to avoid making outbound API calls in CI + k8s_versions = ["1.18", "1.19", "1.20"] + m = Mock() + m.return_value = k8s_versions + if grab_latest_version: + m.return_value = k8s_versions[-1] + return m + + if cloud_provider == "aws": + monkeypatch.setattr( + "_nebari.utils.amazon_web_services.kubernetes_versions", + _mock_kubernetes_versions(), + ) + elif cloud_provider == "azure": + monkeypatch.setattr( + "_nebari.utils.azure_cloud.kubernetes_versions", + _mock_kubernetes_versions(), + ) + elif cloud_provider == "do": + monkeypatch.setattr( + "_nebari.utils.digital_ocean.kubernetes_versions", + _mock_kubernetes_versions(), + ) + elif cloud_provider == "gcp": + monkeypatch.setattr( + "_nebari.utils.google_cloud.kubernetes_versions", + _mock_kubernetes_versions(), + ) + + output_directory = tmp_path / f"{cloud_provider}_output_dir" + output_directory.mkdir() + nebari_config_loc = output_directory / NEBARI_CONFIG_FN + + # data that should NOT be deleted when `nebari render` is called + # see test_render.py::test_remove_existing_renders + preserved_directory = output_directory / PRESERVED_DIR + preserved_directory.mkdir() + preserved_filename = preserved_directory / "file.txt" + preserved_filename.write_text("This is a test...") + + yield (nebari_config_loc, render_config_inputs) diff --git a/tests/tests_unit/notebooks/test-ipython-basic.ipynb b/tests/tests_unit/notebooks/test-ipython-basic.ipynb new file mode 100644 index 000000000..e69de29bb diff --git a/tests/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml similarity index 100% rename from tests/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml rename to tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml diff --git a/tests/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml similarity index 100% rename from tests/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml rename to tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml diff --git a/tests/qhub-config-yaml-files-for-upgrade/qhub-users-import.json b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json similarity index 100% rename from tests/qhub-config-yaml-files-for-upgrade/qhub-users-import.json rename to tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json diff --git a/tests/scripts/minikube-loadbalancer-ip.py b/tests/tests_unit/scripts/minikube-loadbalancer-ip.py similarity index 100% rename from tests/scripts/minikube-loadbalancer-ip.py rename to tests/tests_unit/scripts/minikube-loadbalancer-ip.py diff --git a/tests/test_cli.py b/tests/tests_unit/test_cli.py similarity index 100% rename from tests/test_cli.py rename to tests/tests_unit/test_cli.py diff --git a/tests/test_commons.py b/tests/tests_unit/test_commons.py similarity index 100% rename from tests/test_commons.py rename to tests/tests_unit/test_commons.py diff --git a/tests/test_dependencies.py b/tests/tests_unit/test_dependencies.py similarity index 97% rename from tests/test_dependencies.py rename to tests/tests_unit/test_dependencies.py index f04f36526..cb5629e71 100644 --- a/tests/test_dependencies.py +++ b/tests/tests_unit/test_dependencies.py @@ -3,7 +3,7 @@ import pytest -SRC_DIR = Path(__file__).parent.parent +SRC_DIR = Path(__file__).parent.parent.parent PYPROJECT = SRC_DIR / "pyproject.toml" diff --git a/tests/test_init.py b/tests/tests_unit/test_init.py similarity index 100% rename from tests/test_init.py rename to tests/tests_unit/test_init.py diff --git a/tests/test_links.py b/tests/tests_unit/test_links.py similarity index 100% rename from tests/test_links.py rename to tests/tests_unit/test_links.py diff --git a/tests/test_render.py b/tests/tests_unit/test_render.py similarity index 100% rename from tests/test_render.py rename to tests/tests_unit/test_render.py diff --git a/tests/test_schema.py b/tests/tests_unit/test_schema.py similarity index 100% rename from tests/test_schema.py rename to tests/tests_unit/test_schema.py diff --git a/tests/test_upgrade.py b/tests/tests_unit/test_upgrade.py similarity index 100% rename from tests/test_upgrade.py rename to tests/tests_unit/test_upgrade.py diff --git a/tests/tests_unit/utils.py b/tests/tests_unit/utils.py new file mode 100644 index 000000000..82dffdcd3 --- /dev/null +++ b/tests/tests_unit/utils.py @@ -0,0 +1,25 @@ +from functools import partial + +from _nebari.initialize import render_config + +DEFAULT_TERRAFORM_STATE = "remote" + +DEFAULT_GH_REPO = "github.com/test/test" +render_config_partial = partial( + render_config, + repository=DEFAULT_GH_REPO, + repository_auto_provision=False, + auth_auto_provision=False, + terraform_state=DEFAULT_TERRAFORM_STATE, + disable_prompt=True, +) +INIT_INPUTS = [ + # project, namespace, domain, cloud_provider, ci_provider, auth_provider + ("pytestdo", "dev", "do.nebari.dev", "do", "github-actions", "github"), + ("pytestaws", "dev", "aws.nebari.dev", "aws", "github-actions", "github"), + ("pytestgcp", "dev", "gcp.nebari.dev", "gcp", "github-actions", "github"), + ("pytestazure", "dev", "azure.nebari.dev", "azure", "github-actions", "github"), +] + +NEBARI_CONFIG_FN = "nebari-config.yaml" +PRESERVED_DIR = "preserved_dir" diff --git a/tests/vale/styles/vocab.txt b/tests/tests_unit/vale/styles/vocab.txt similarity index 100% rename from tests/vale/styles/vocab.txt rename to tests/tests_unit/vale/styles/vocab.txt diff --git a/tests_e2e/playwright/run_notebook.py b/tests_e2e/playwright/run_notebook.py deleted file mode 100644 index 12d2e2038..000000000 --- a/tests_e2e/playwright/run_notebook.py +++ /dev/null @@ -1,82 +0,0 @@ -import contextlib -import logging -from pathlib import Path - -from navigator import Navigator - -logger = logging.getLogger() - - -class Notebook: - def __init__(self, navigator: Navigator): - self.nav = navigator - self.nav.initialize - - def run(self, path, expected_output_text, conda_env, runtime=30000, retry=2): - """Run jupyter notebook and check for expected output text anywhere on - the page. - - Note: This will look for and exact match of expected_output_text - _anywhere_ on the page so be sure that your text is unique. - - Conda environments may still be being built shortly after deployment. - - conda_env: str - Name of conda environment. Python conda environments have the - structure "conda-env-nebari-git-nebari-git-dashboard-py" where - the actual name of the environment is "dashboard". - """ - logger.debug(f">>> Running notebook: {path}") - filename = Path(path).name - - # navigate to specific notebook - file_locator = self.nav.page.get_by_text("File", exact=True) - - file_locator.wait_for( - timeout=self.nav.wait_for_server_spinup, - state="attached", - ) - file_locator.click() - self.nav.page.get_by_role("menuitem", name="Open from Path…").get_by_text( - "Open from Path…" - ).click() - self.nav.page.get_by_placeholder("/path/relative/to/jlab/root").fill(path) - self.nav.page.get_by_role("button", name="Open", exact=True).click() - # give the page a second to open, otherwise the options in the kernel - # menu will be disabled. - self.nav.page.wait_for_load_state("networkidle") - if self.nav.page.get_by_text( - "Could not find path:", - exact=False, - ).is_visible(): - logger.debug("Path to notebook is invalid") - raise RuntimeError("Path to notebook is invalid") - # make sure the focus is on the dashboard tab we want to run - self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() - self.nav.set_environment(kernel=conda_env) - - # make sure that this notebook is one currently selected - self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() - - for i in range(retry): - self._restart_run_all() - - output_locator = self.nav.page.get_by_text(expected_output_text, exact=True) - with contextlib.suppress(Exception): - if output_locator.is_visible(): - break - - def _restart_run_all(self): - # restart run all cells - self.nav.page.get_by_text("Kernel", exact=True).click() - self.nav.page.get_by_role( - "menuitem", name="Restart Kernel and Run All Cells…" - ).get_by_text("Restart Kernel and Run All Cells…").click() - - # Restart dialog appears most, but not all of the time (e.g. set - # No Kernel, then Restart Run All) - restart_dialog_button = self.nav.page.get_by_role( - "button", name="Restart", exact=True - ) - if restart_dialog_button.is_visible(): - restart_dialog_button.click() diff --git a/tests_e2e/playwright/test_playwright.py b/tests_e2e/playwright/test_playwright.py deleted file mode 100644 index 264243fd9..000000000 --- a/tests_e2e/playwright/test_playwright.py +++ /dev/null @@ -1,14 +0,0 @@ -from run_notebook import Notebook - - -def test_notebook(navigator, test_data_root): - test_app = Notebook(navigator=navigator) - notebook_name = "test_notebook_output.ipynb" - with open(test_data_root / notebook_name, "r") as notebook: - test_app.nav.write_file(filepath=notebook_name, content=notebook.read()) - test_app.run( - path=notebook_name, - expected_output_text="success: 6", - conda_env="conda-env-default-py", - runtime=60000, - ) diff --git a/tests_integration/conftest.py b/tests_integration/conftest.py deleted file mode 100644 index 6a64a20ab..000000000 --- a/tests_integration/conftest.py +++ /dev/null @@ -1 +0,0 @@ -pytest_plugins = ["tests_integration.deployment_fixtures"] diff --git a/tests_integration/deployment_fixtures.py b/tests_integration/deployment_fixtures.py deleted file mode 100644 index bbf7190a5..000000000 --- a/tests_integration/deployment_fixtures.py +++ /dev/null @@ -1,95 +0,0 @@ -import logging -import os -import random -import string -import warnings -from pathlib import Path - -import pytest -import yaml -from urllib3.exceptions import InsecureRequestWarning - -from _nebari.deploy import deploy_configuration -from _nebari.destroy import destroy_configuration -from _nebari.render import render_template -from tests.utils import render_config_partial - -DEPLOYMENT_DIR = "_test_deploy" - -logger = logging.getLogger(__name__) - - -def ignore_warnings(): - # Ignore this for now, as test is failing due to a - # DeprecationWarning and InsecureRequestWarning - warnings.filterwarnings("ignore", category=DeprecationWarning) - warnings.filterwarnings("ignore", category=InsecureRequestWarning) - - -def _random_letters(length=5): - letters = string.ascii_letters - return "".join(random.choice(letters) for _ in range(length)).lower() - - -def _get_or_create_deployment_directory(cloud): - """This will create a directory to initialise and deploy - Nebari from. - """ - deployment_dirs = list(Path(Path(DEPLOYMENT_DIR) / cloud).glob(f"pytest{cloud}*")) - if deployment_dirs: - deployment_dir = deployment_dirs[0] - else: - project_name = f"pytest{cloud}{_random_letters()}" - deployment_dir = Path(Path(Path(DEPLOYMENT_DIR) / cloud) / project_name) - deployment_dir.mkdir(parents=True) - return deployment_dir - - -def _set_do_environment(): - os.environ["AWS_ACCESS_KEY_ID"] = os.environ["SPACES_ACCESS_KEY_ID"] - os.environ["AWS_SECRET_ACCESS_KEY"] = os.environ["SPACES_SECRET_ACCESS_KEY"] - - -@pytest.fixture(scope="session") -def deploy(request): - """Deploy Nebari on the given cloud, currently only DigitalOcean""" - ignore_warnings() - cloud = request.param - _set_do_environment() - deployment_dir = _get_or_create_deployment_directory(cloud) - config = render_config_partial( - project_name=deployment_dir.name, - namespace="dev", - nebari_domain=f"ci-{cloud}.nebari.dev", - cloud_provider=cloud, - ci_provider="github-actions", - auth_provider="github", - ) - deployment_dir_abs = deployment_dir.absolute() - os.chdir(deployment_dir) - logger.info(f"Temporary directory: {deployment_dir}") - with open(Path("nebari-config.yaml"), "w") as f: - yaml.dump(config, f) - render_template(deployment_dir_abs, Path("nebari-config.yaml")) - try: - yield deploy_configuration( - config=config, - dns_provider="cloudflare", - dns_auto_provision=True, - disable_prompt=True, - disable_checks=False, - skip_remote_state_provision=False, - ) - except Exception as e: - logger.info(f"Deploy Failed, Exception: {e}") - logger.exception(e) - logger.info("Tearing down") - return _destroy(config) - - -def _destroy(config): - destroy_configuration(config) - - -def on_cloud(param): - return pytest.mark.parametrize("deploy", [param], indirect=True)