From ae811297133daec81b7a60f866480607de49817d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:53:11 +0100 Subject: [PATCH 1/6] remove digital ocean tests --- .github/ISSUE_TEMPLATE/bug-report.yml | 1 - .github/workflows/test-provider.yaml | 2 -- README.md | 2 +- tests/tests_integration/test_all_clouds.py | 1 - tests/tests_unit/cli_validate/do.happy.yaml | 3 --- tests/tests_unit/conftest.py | 16 ---------------- tests/tests_unit/test_cli_init.py | 2 -- tests/tests_unit/test_cli_validate.py | 2 -- tests/tests_unit/test_schema.py | 5 ----- 9 files changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index a89ff5e880..328cc6b147 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -93,7 +93,6 @@ body: - "Azure" - "GCP" - "AWS" - - "Digital Ocean" - "kind" validations: required: false diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index fef1254495..b56eae90b9 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -49,7 +49,6 @@ jobs: provider: - aws - azure - - do - gcp - local - existing @@ -88,7 +87,6 @@ jobs: kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID; kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID; kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID; - kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN; kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_ACCESS_KEY_ID | SPACES_ACCESS_KEY_ID; kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_SECRET_ACCESS_KEY | SPACES_SECRET_ACCESS_KEY; diff --git a/README.md b/README.md index 5dcce91258..197c2013cc 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Hence, it is not necessary to know any of the technologies mentioned above to ha ### Cloud Providers ☁️ -Nebari offers out-of-the-box support for the major public cloud providers: [Digital Ocean](https://www.digitalocean.com/), +Nebari offers out-of-the-box support for the major public cloud providers: Amazon [AWS](https://aws.amazon.com/), [GCP](https://cloud.google.com/ "Google Cloud Provider"), and Microsoft [Azure](https://azure.microsoft.com/en-us/). ![High-level illustration of Nebari architecture](https://raw.githubusercontent.com/nebari-dev/nebari-docs/main/docs/static/img/welcome/nebari_overview_sequence.png) diff --git a/tests/tests_integration/test_all_clouds.py b/tests/tests_integration/test_all_clouds.py index 8a163fb7b6..6a9bf87dd4 100644 --- a/tests/tests_integration/test_all_clouds.py +++ b/tests/tests_integration/test_all_clouds.py @@ -2,7 +2,6 @@ def test_service_status(deploy): - """Tests if deployment on DigitalOcean succeeds""" service_urls = deploy["stages/07-kubernetes-services"]["service_urls"]["value"] assert ( requests.get(service_urls["jupyterhub"]["health_url"], verify=False).status_code diff --git a/tests/tests_unit/cli_validate/do.happy.yaml b/tests/tests_unit/cli_validate/do.happy.yaml index 4ca0b2e62f..0ec2a7b528 100644 --- a/tests/tests_unit/cli_validate/do.happy.yaml +++ b/tests/tests_unit/cli_validate/do.happy.yaml @@ -23,6 +23,3 @@ theme: certificate: type: lets-encrypt acme_email: test@example.com -digital_ocean: - kubernetes_version: '1.20.2-do.0' - region: nyc3 diff --git a/tests/tests_unit/conftest.py b/tests/tests_unit/conftest.py index ce60e44799..82e1d42269 100644 --- a/tests/tests_unit/conftest.py +++ b/tests/tests_unit/conftest.py @@ -63,22 +63,6 @@ def _mock_return_value(return_value): "1.20", ], "_nebari.provider.cloud.azure_cloud.check_credentials": None, - # Digital Ocean - "_nebari.provider.cloud.digital_ocean.kubernetes_versions": [ - "1.19.2-do.3", - "1.20.2-do.0", - "1.21.5-do.0", - ], - "_nebari.provider.cloud.digital_ocean.check_credentials": None, - "_nebari.provider.cloud.digital_ocean.regions": [ - {"name": "New York 3", "slug": "nyc3"}, - ], - "_nebari.provider.cloud.digital_ocean.instances": [ - {"name": "s-2vcpu-4gb", "slug": "s-2vcpu-4gb"}, - {"name": "g-2vcpu-8gb", "slug": "g-2vcpu-8gb"}, - {"name": "g-8vcpu-32gb", "slug": "g-8vcpu-32gb"}, - {"name": "g-4vcpu-16gb", "slug": "g-4vcpu-16gb"}, - ], # Google Cloud "_nebari.provider.cloud.google_cloud.kubernetes_versions": [ "1.18", diff --git a/tests/tests_unit/test_cli_init.py b/tests/tests_unit/test_cli_init.py index 9afab5ddc5..4fe0141c86 100644 --- a/tests/tests_unit/test_cli_init.py +++ b/tests/tests_unit/test_cli_init.py @@ -265,8 +265,6 @@ def get_provider_section_header(provider: str): return "google_cloud_platform" if provider == "azure": return "azure" - if provider == "do": - return "digital_ocean" return "" diff --git a/tests/tests_unit/test_cli_validate.py b/tests/tests_unit/test_cli_validate.py index faf2efa8a1..b12d3cfea0 100644 --- a/tests/tests_unit/test_cli_validate.py +++ b/tests/tests_unit/test_cli_validate.py @@ -221,7 +221,6 @@ def test_cli_validate_error_from_env( } }, ), - ("do", {"digital_ocean": {"kubernetes_version": "1.20", "region": "nyc3"}}), pytest.param( "local", {"security": {"authentication": {"type": "Auth0"}}}, @@ -248,7 +247,6 @@ def test_cli_validate_error_missing_cloud_env( "ARM_TENANT_ID", "ARM_CLIENT_ID", "ARM_CLIENT_SECRET", - "DIGITALOCEAN_TOKEN", "SPACES_ACCESS_KEY_ID", "SPACES_SECRET_ACCESS_KEY", "AUTH0_CLIENT_ID", diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index fa6a0c747c..b88a32eec7 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -102,11 +102,6 @@ def test_provider_validation(config_schema, provider, exception): "kubernetes_version": "1.18", }, ), - ( - "do", - "digital_ocean", - {"region": "nyc3", "kubernetes_version": "1.19.2-do.3"}, - ), ( "azure", "azure", From 6470330bc83fd253e17d0c3ea3dab01a18f1636a Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:23:41 +0100 Subject: [PATCH 2/6] remove do unit tests --- tests/tests_unit/test_cli_init.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/tests_unit/test_cli_init.py b/tests/tests_unit/test_cli_init.py index 4fe0141c86..03b22557ae 100644 --- a/tests/tests_unit/test_cli_init.py +++ b/tests/tests_unit/test_cli_init.py @@ -17,13 +17,11 @@ "aws": ["1.20"], "azure": ["1.20"], "gcp": ["1.20"], - "do": ["1.21.5-do.0"], } MOCK_CLOUD_REGIONS = { "aws": ["us-east-1"], "azure": [AZURE_DEFAULT_REGION], "gcp": ["us-central1"], - "do": ["nyc3"], } @@ -70,7 +68,7 @@ def generate_test_data_test_cli_init_happy_path(): """ test_data = [] - for provider in ["local", "aws", "azure", "gcp", "do", "existing"]: + for provider in ["local", "aws", "azure", "gcp", "existing"]: for region in get_cloud_regions(provider): for project_name in ["testproject"]: for domain_name in [f"{project_name}.example.com"]: @@ -265,7 +263,6 @@ def get_provider_section_header(provider: str): return "google_cloud_platform" if provider == "azure": return "azure" - return "" @@ -276,8 +273,6 @@ def get_cloud_regions(provider: str): return MOCK_CLOUD_REGIONS["gcp"] if provider == "azure": return MOCK_CLOUD_REGIONS["azure"] - if provider == "do": - return MOCK_CLOUD_REGIONS["do"] return "" @@ -289,7 +284,4 @@ def get_kubernetes_versions(provider: str): return MOCK_KUBERNETES_VERSIONS["gcp"] if provider == "azure": return MOCK_KUBERNETES_VERSIONS["azure"] - if provider == "do": - return MOCK_KUBERNETES_VERSIONS["do"] - return "" From 6fabb877fa0bb16a6f48aafba4eeb9c867bf677d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:32:49 +0100 Subject: [PATCH 3/6] remove do tests --- .../qhub-config-do-310-customauth.yaml | 131 ------------- .../qhub-config-do-310.yaml | 128 ------------- .../qhub-users-import.json | 15 -- tests/tests_unit/test_upgrade.py | 175 ------------------ 4 files changed, 449 deletions(-) delete mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml delete mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml delete mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json delete mode 100644 tests/tests_unit/test_upgrade.py diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml deleted file mode 100644 index 50a2b89af4..0000000000 --- a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml +++ /dev/null @@ -1,131 +0,0 @@ -project_name: do-pytest -provider: do -domain: do.nebari.dev -certificate: - type: self-signed -security: - authentication: - type: custom - authentication_class: 'firstuseauthenticator.FirstUseAuthenticator' - config: - min_password_length: 5 - users: - example-user: - uid: 1000 - primary_group: admin - secondary_groups: - - users - password: $2b$12$YrEkTAEFfo4fKO7lYPpReegKagd1irrW5YmRugJcaPCjkVaPzrVLq - groups: - users: - gid: 100 - admin: - gid: 101 -default_images: - jupyterhub: quansight/nebari-jupyterhub:v0.3.10 - jupyterlab: quansight/nebari-jupyterlab:v0.3.10 - dask_worker: quansight/nebari-dask-worker:v0.3.10 - dask_gateway: quansight/nebari-dask-gateway:v0.3.10 -storage: - conda_store: 60Gi - shared_filesystem: 100Gi -theme: - jupyterhub: - hub_title: Nebari - do-pytest - hub_subtitle: Autoscaling Compute Environment on Digital Ocean - welcome: Welcome to do.nebari.dev. It is maintained by Quansight - staff. The hub's configuration is stored in a github repository based on - https://github.com/Quansight/nebari/. - To provide feedback and report any technical problems, please use the github - issue tracker. - logo: /hub/custom/images/jupyter_nebari_logo.svg - primary_color: '#4f4173' - secondary_color: '#957da6' - accent_color: '#32C574' - text_color: '#111111' - h1_color: '#652e8e' - h2_color: '#652e8e' -terraform_state: - type: remote -namespace: dev -digital_ocean: - region: nyc3 - kubernetes_version: 1.21.5-do.0 - node_groups: - general: - instance: s-2vcpu-4gb - min_nodes: 1 - max_nodes: 1 - user: - instance: g-2vcpu-8gb - min_nodes: 1 - max_nodes: 5 - worker: - instance: g-2vcpu-8gb - min_nodes: 1 - max_nodes: 5 -profiles: - jupyterlab: - - display_name: Small Instance - description: Stable environment with 1 cpu / 4 GB ram - default: true - kubespawner_override: - cpu_limit: 1 - cpu_guarantee: 0.75 - mem_limit: 4G - mem_guarantee: 2.5G - image: quansight/nebari-jupyterlab:v0.3.10 - - display_name: Medium Instance - description: Stable environment with 2 cpu / 8 GB ram - kubespawner_override: - cpu_limit: 2 - cpu_guarantee: 1.5 - mem_limit: 8G - mem_guarantee: 5G - image: quansight/nebari-jupyterlab:v0.3.10 - dask_worker: - Small Worker: - worker_cores_limit: 1 - worker_cores: 0.75 - worker_memory_limit: 4G - worker_memory: 2.5G - worker_threads: 1 - image: quansight/nebari-dask-worker:v0.3.10 - Medium Worker: - worker_cores_limit: 2 - worker_cores: 1.5 - worker_memory_limit: 8G - worker_memory: 5G - worker_threads: 2 - image: quansight/nebari-dask-worker:v0.3.10 -environments: - environment-dask.yaml: - name: dask - channels: - - conda-forge - dependencies: - - python - - ipykernel - - ipywidgets - - python-graphviz - - dask ==2.30.0 - - distributed ==2.30.1 - - dask-gateway ==0.9.0 - - numpy - - numba - - pandas - environment-dashboard.yaml: - name: dashboard - channels: - - conda-forge - dependencies: - - python - - ipykernel - - ipywidgets >=7.6 - - param - - python-graphviz - - matplotlib >=3.3.4 - - panel >=0.10.3 - - voila >=0.2.7 - - streamlit >=0.76 - - dash >=1.19 diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml deleted file mode 100644 index a3a06da6a2..0000000000 --- a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml +++ /dev/null @@ -1,128 +0,0 @@ -project_name: do-pytest -provider: do -domain: do.nebari.dev -certificate: - type: self-signed -security: - authentication: - type: password - users: - example-user: - uid: 1000 - primary_group: admin - secondary_groups: - - users - password: $2b$12$YrEkTAEFfo4fKO7lYPpReegKagd1irrW5YmRugJcaPCjkVaPzrVLq - groups: - users: - gid: 100 - admin: - gid: 101 -default_images: - jupyterhub: quansight/nebari-jupyterhub:v0.3.10 - jupyterlab: quansight/nebari-jupyterlab:v0.3.10 - dask_worker: quansight/nebari-dask-worker:v0.3.10 - dask_gateway: quansight/nebari-dask-gateway:v0.3.10 -storage: - conda_store: 60Gi - shared_filesystem: 100Gi -theme: - jupyterhub: - hub_title: Nebari - do-pytest - hub_subtitle: Autoscaling Compute Environment on Digital Ocean - welcome: Welcome to do.nebari.dev. It is maintained by Quansight - staff. The hub's configuration is stored in a github repository based on - https://github.com/Quansight/nebari/. - To provide feedback and report any technical problems, please use the github - issue tracker. - logo: /hub/custom/images/jupyter_nebari_logo.svg - primary_color: '#4f4173' - secondary_color: '#957da6' - accent_color: '#32C574' - text_color: '#111111' - h1_color: '#652e8e' - h2_color: '#652e8e' -terraform_state: - type: remote -namespace: dev -digital_ocean: - region: nyc3 - kubernetes_version: 1.21.5-do.0 - node_groups: - general: - instance: s-2vcpu-4gb - min_nodes: 1 - max_nodes: 1 - user: - instance: g-2vcpu-8gb - min_nodes: 1 - max_nodes: 5 - worker: - instance: g-2vcpu-8gb - min_nodes: 1 - max_nodes: 5 -profiles: - jupyterlab: - - display_name: Small Instance - description: Stable environment with 1 cpu / 4 GB ram - default: true - kubespawner_override: - cpu_limit: 1 - cpu_guarantee: 0.75 - mem_limit: 4G - mem_guarantee: 2.5G - image: quansight/nebari-jupyterlab:v0.3.10 - - display_name: Medium Instance - description: Stable environment with 2 cpu / 8 GB ram - kubespawner_override: - cpu_limit: 2 - cpu_guarantee: 1.5 - mem_limit: 8G - mem_guarantee: 5G - image: quansight/nebari-jupyterlab:v0.3.10 - dask_worker: - Small Worker: - worker_cores_limit: 1 - worker_cores: 0.75 - worker_memory_limit: 4G - worker_memory: 2.5G - worker_threads: 1 - image: quansight/nebari-dask-worker:v0.3.10 - Medium Worker: - worker_cores_limit: 2 - worker_cores: 1.5 - worker_memory_limit: 8G - worker_memory: 5G - worker_threads: 2 - image: quansight/nebari-dask-worker:v0.3.10 -environments: - environment-dask.yaml: - name: dask - channels: - - conda-forge - dependencies: - - python - - ipykernel - - ipywidgets - - python-graphviz - - dask ==2.30.0 - - distributed ==2.30.1 - - dask-gateway ==0.9.0 - - numpy - - numba - - pandas - environment-dashboard.yaml: - name: dashboard - channels: - - conda-forge - dependencies: - - python - - ipykernel - - ipywidgets >=7.6 - - param - - python-graphviz - - matplotlib >=3.3.4 - - panel >=0.10.3 - - voila >=0.2.7 - - streamlit >=0.76 - - dash >=1.19 diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json deleted file mode 100644 index 0bef9eb17c..0000000000 --- a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "nebari", - "realm": "nebari", - "users": [ - { - "username": "example-user", - "enabled": true, - "groups": [ - "admin", - "users" - ] - } - ], - "groups": [] -} diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py deleted file mode 100644 index f6e3f80348..0000000000 --- a/tests/tests_unit/test_upgrade.py +++ /dev/null @@ -1,175 +0,0 @@ -from contextlib import nullcontext -from pathlib import Path - -import pytest -from rich.prompt import Prompt - -from _nebari.upgrade import do_upgrade -from _nebari.version import __version__, rounded_ver_parse -from nebari.plugins import nebari_plugin_manager - - -@pytest.fixture -def qhub_users_import_json(): - return ( - ( - Path(__file__).parent - / "./qhub-config-yaml-files-for-upgrade/qhub-users-import.json" - ) - .read_text() - .rstrip() - ) - - -@pytest.mark.parametrize( - "old_qhub_config_path_str,attempt_fixes,expect_upgrade_error", - [ - ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml", - False, - False, - ), - ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", - False, - True, - ), - ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", - True, - False, - ), - ], -) -def test_upgrade_4_0( - old_qhub_config_path_str, - attempt_fixes, - expect_upgrade_error, - tmp_path, - qhub_users_import_json, - monkeypatch, -): - - def mock_input(prompt, **kwargs): - # Mock different upgrade steps prompt answers - if ( - prompt - == "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] " - ): - return "y" - elif ( - prompt - == "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually." - ): - return "N" - elif ( - prompt - == "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?" - ): - return "y" - elif ( - prompt - == "[bold]Would you like Nebari to assign the corresponding role to all of your current groups automatically?[/bold]" - ): - return "N" - # All other prompts will be answered with "y" - else: - return "y" - - monkeypatch.setattr(Prompt, "ask", mock_input) - - old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str - - tmp_qhub_config = Path(tmp_path, old_qhub_config_path.name) - tmp_qhub_config.write_text(old_qhub_config_path.read_text()) # Copy contents to tmp - - orig_contents = tmp_qhub_config.read_text() # Read in initial contents - - assert not Path(tmp_path, "qhub-users-import.json").exists() - - # Do the upgrade - if not expect_upgrade_error: - do_upgrade( - tmp_qhub_config, attempt_fixes - ) # Would raise an error if invalid by current Nebari version's standards - else: - with pytest.raises(ValueError): - do_upgrade(tmp_qhub_config, attempt_fixes) - return - - # Check the resulting YAML - config = nebari_plugin_manager.read_config(tmp_qhub_config) - - assert len(config.security.keycloak.initial_root_password) == 16 - assert not hasattr(config.security, "users") - assert not hasattr(config.security, "groups") - - __rounded_version__ = rounded_ver_parse(__version__) - - # Check image versions have been bumped up - assert ( - config.default_images.jupyterhub - == f"quansight/nebari-jupyterhub:v{__rounded_version__}" - ) - assert ( - config.profiles.jupyterlab[0].kubespawner_override.image - == f"quansight/nebari-jupyterlab:v{__rounded_version__}" - ) - assert config.security.authentication.type != "custom" - - # Keycloak import users json - assert ( - Path(tmp_path, "nebari-users-import.json").read_text().rstrip() - == qhub_users_import_json - ) - - # Check backup - tmp_qhub_config_backup = Path(tmp_path, f"{old_qhub_config_path.name}.old.backup") - - assert orig_contents == tmp_qhub_config_backup.read_text() - - -@pytest.mark.parametrize( - "version_str, exception", - [ - ("1.0.0", nullcontext()), - ("1.cool.0", pytest.raises(ValueError, match=r"Invalid version string .*")), - ("0,1.0", pytest.raises(ValueError, match=r"Invalid version string .*")), - ("", pytest.raises(ValueError, match=r"Invalid version string .*")), - ( - "1.0.0-rc1", - pytest.raises( - AssertionError, - match=r"Invalid version .*: must be a full release version, not a dev/prerelease/postrelease version", - ), - ), - ( - "1.0.0dev1", - pytest.raises( - AssertionError, - match=r"Invalid version .*: must be a full release version, not a dev/prerelease/postrelease version", - ), - ), - ], -) -def test_version_string(new_upgrade_cls, version_str, exception): - with exception: - - class DummyUpgrade(new_upgrade_cls): - version = version_str - - -def test_duplicated_version(new_upgrade_cls): - duplicated_version = "1.2.3" - with pytest.raises( - AssertionError, match=rf"Duplicate UpgradeStep version {duplicated_version}" - ): - - class DummyUpgrade(new_upgrade_cls): - version = duplicated_version - - class DummyUpgrade2(new_upgrade_cls): - version = duplicated_version - - class DummyUpgrade3(new_upgrade_cls): - version = "1.2.4" From 712fc81950d621fb1d3553bf64f57b09ec582366 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:57:09 +0100 Subject: [PATCH 4/6] remove do tests --- tests/tests_unit/conftest.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/tests_unit/conftest.py b/tests/tests_unit/conftest.py index 82e1d42269..7744bb28a9 100644 --- a/tests/tests_unit/conftest.py +++ b/tests/tests_unit/conftest.py @@ -7,7 +7,6 @@ from _nebari.constants import ( AWS_DEFAULT_REGION, AZURE_DEFAULT_REGION, - DO_DEFAULT_REGION, GCP_DEFAULT_REGION, ) from _nebari.initialize import render_config @@ -85,15 +84,6 @@ def _mock_return_value(return_value): @pytest.fixture( params=[ # project, namespace, domain, cloud_provider, region, ci_provider, auth_provider - ( - "pytestdo", - "dev", - "do.nebari.dev", - schema.ProviderEnum.do, - DO_DEFAULT_REGION, - CiEnum.github_actions, - AuthenticationEnum.password, - ), ( "pytestaws", "dev", From b7c4d544f042b62ebcd62a74674893d69b07806f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:41:41 +0100 Subject: [PATCH 5/6] restore needed test files --- .../qhub-config-do-310-customauth.yaml | 131 +++++++++++++ .../qhub-config-do-310.yaml | 128 +++++++++++++ .../qhub-users-import.json | 15 ++ tests/tests_unit/test_upgrade.py | 175 ++++++++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml create mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml create mode 100644 tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json create mode 100644 tests/tests_unit/test_upgrade.py diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml new file mode 100644 index 0000000000..50a2b89af4 --- /dev/null +++ b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml @@ -0,0 +1,131 @@ +project_name: do-pytest +provider: do +domain: do.nebari.dev +certificate: + type: self-signed +security: + authentication: + type: custom + authentication_class: 'firstuseauthenticator.FirstUseAuthenticator' + config: + min_password_length: 5 + users: + example-user: + uid: 1000 + primary_group: admin + secondary_groups: + - users + password: $2b$12$YrEkTAEFfo4fKO7lYPpReegKagd1irrW5YmRugJcaPCjkVaPzrVLq + groups: + users: + gid: 100 + admin: + gid: 101 +default_images: + jupyterhub: quansight/nebari-jupyterhub:v0.3.10 + jupyterlab: quansight/nebari-jupyterlab:v0.3.10 + dask_worker: quansight/nebari-dask-worker:v0.3.10 + dask_gateway: quansight/nebari-dask-gateway:v0.3.10 +storage: + conda_store: 60Gi + shared_filesystem: 100Gi +theme: + jupyterhub: + hub_title: Nebari - do-pytest + hub_subtitle: Autoscaling Compute Environment on Digital Ocean + welcome: Welcome to do.nebari.dev. It is maintained by Quansight + staff. The hub's configuration is stored in a github repository based on + https://github.com/Quansight/nebari/. + To provide feedback and report any technical problems, please use the github + issue tracker. + logo: /hub/custom/images/jupyter_nebari_logo.svg + primary_color: '#4f4173' + secondary_color: '#957da6' + accent_color: '#32C574' + text_color: '#111111' + h1_color: '#652e8e' + h2_color: '#652e8e' +terraform_state: + type: remote +namespace: dev +digital_ocean: + region: nyc3 + kubernetes_version: 1.21.5-do.0 + node_groups: + general: + instance: s-2vcpu-4gb + min_nodes: 1 + max_nodes: 1 + user: + instance: g-2vcpu-8gb + min_nodes: 1 + max_nodes: 5 + worker: + instance: g-2vcpu-8gb + min_nodes: 1 + max_nodes: 5 +profiles: + jupyterlab: + - display_name: Small Instance + description: Stable environment with 1 cpu / 4 GB ram + default: true + kubespawner_override: + cpu_limit: 1 + cpu_guarantee: 0.75 + mem_limit: 4G + mem_guarantee: 2.5G + image: quansight/nebari-jupyterlab:v0.3.10 + - display_name: Medium Instance + description: Stable environment with 2 cpu / 8 GB ram + kubespawner_override: + cpu_limit: 2 + cpu_guarantee: 1.5 + mem_limit: 8G + mem_guarantee: 5G + image: quansight/nebari-jupyterlab:v0.3.10 + dask_worker: + Small Worker: + worker_cores_limit: 1 + worker_cores: 0.75 + worker_memory_limit: 4G + worker_memory: 2.5G + worker_threads: 1 + image: quansight/nebari-dask-worker:v0.3.10 + Medium Worker: + worker_cores_limit: 2 + worker_cores: 1.5 + worker_memory_limit: 8G + worker_memory: 5G + worker_threads: 2 + image: quansight/nebari-dask-worker:v0.3.10 +environments: + environment-dask.yaml: + name: dask + channels: + - conda-forge + dependencies: + - python + - ipykernel + - ipywidgets + - python-graphviz + - dask ==2.30.0 + - distributed ==2.30.1 + - dask-gateway ==0.9.0 + - numpy + - numba + - pandas + environment-dashboard.yaml: + name: dashboard + channels: + - conda-forge + dependencies: + - python + - ipykernel + - ipywidgets >=7.6 + - param + - python-graphviz + - matplotlib >=3.3.4 + - panel >=0.10.3 + - voila >=0.2.7 + - streamlit >=0.76 + - dash >=1.19 diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml new file mode 100644 index 0000000000..a3a06da6a2 --- /dev/null +++ b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml @@ -0,0 +1,128 @@ +project_name: do-pytest +provider: do +domain: do.nebari.dev +certificate: + type: self-signed +security: + authentication: + type: password + users: + example-user: + uid: 1000 + primary_group: admin + secondary_groups: + - users + password: $2b$12$YrEkTAEFfo4fKO7lYPpReegKagd1irrW5YmRugJcaPCjkVaPzrVLq + groups: + users: + gid: 100 + admin: + gid: 101 +default_images: + jupyterhub: quansight/nebari-jupyterhub:v0.3.10 + jupyterlab: quansight/nebari-jupyterlab:v0.3.10 + dask_worker: quansight/nebari-dask-worker:v0.3.10 + dask_gateway: quansight/nebari-dask-gateway:v0.3.10 +storage: + conda_store: 60Gi + shared_filesystem: 100Gi +theme: + jupyterhub: + hub_title: Nebari - do-pytest + hub_subtitle: Autoscaling Compute Environment on Digital Ocean + welcome: Welcome to do.nebari.dev. It is maintained by Quansight + staff. The hub's configuration is stored in a github repository based on + https://github.com/Quansight/nebari/. + To provide feedback and report any technical problems, please use the github + issue tracker. + logo: /hub/custom/images/jupyter_nebari_logo.svg + primary_color: '#4f4173' + secondary_color: '#957da6' + accent_color: '#32C574' + text_color: '#111111' + h1_color: '#652e8e' + h2_color: '#652e8e' +terraform_state: + type: remote +namespace: dev +digital_ocean: + region: nyc3 + kubernetes_version: 1.21.5-do.0 + node_groups: + general: + instance: s-2vcpu-4gb + min_nodes: 1 + max_nodes: 1 + user: + instance: g-2vcpu-8gb + min_nodes: 1 + max_nodes: 5 + worker: + instance: g-2vcpu-8gb + min_nodes: 1 + max_nodes: 5 +profiles: + jupyterlab: + - display_name: Small Instance + description: Stable environment with 1 cpu / 4 GB ram + default: true + kubespawner_override: + cpu_limit: 1 + cpu_guarantee: 0.75 + mem_limit: 4G + mem_guarantee: 2.5G + image: quansight/nebari-jupyterlab:v0.3.10 + - display_name: Medium Instance + description: Stable environment with 2 cpu / 8 GB ram + kubespawner_override: + cpu_limit: 2 + cpu_guarantee: 1.5 + mem_limit: 8G + mem_guarantee: 5G + image: quansight/nebari-jupyterlab:v0.3.10 + dask_worker: + Small Worker: + worker_cores_limit: 1 + worker_cores: 0.75 + worker_memory_limit: 4G + worker_memory: 2.5G + worker_threads: 1 + image: quansight/nebari-dask-worker:v0.3.10 + Medium Worker: + worker_cores_limit: 2 + worker_cores: 1.5 + worker_memory_limit: 8G + worker_memory: 5G + worker_threads: 2 + image: quansight/nebari-dask-worker:v0.3.10 +environments: + environment-dask.yaml: + name: dask + channels: + - conda-forge + dependencies: + - python + - ipykernel + - ipywidgets + - python-graphviz + - dask ==2.30.0 + - distributed ==2.30.1 + - dask-gateway ==0.9.0 + - numpy + - numba + - pandas + environment-dashboard.yaml: + name: dashboard + channels: + - conda-forge + dependencies: + - python + - ipykernel + - ipywidgets >=7.6 + - param + - python-graphviz + - matplotlib >=3.3.4 + - panel >=0.10.3 + - voila >=0.2.7 + - streamlit >=0.76 + - dash >=1.19 diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json new file mode 100644 index 0000000000..0bef9eb17c --- /dev/null +++ b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-users-import.json @@ -0,0 +1,15 @@ +{ + "id": "nebari", + "realm": "nebari", + "users": [ + { + "username": "example-user", + "enabled": true, + "groups": [ + "admin", + "users" + ] + } + ], + "groups": [] +} diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py new file mode 100644 index 0000000000..f6e3f80348 --- /dev/null +++ b/tests/tests_unit/test_upgrade.py @@ -0,0 +1,175 @@ +from contextlib import nullcontext +from pathlib import Path + +import pytest +from rich.prompt import Prompt + +from _nebari.upgrade import do_upgrade +from _nebari.version import __version__, rounded_ver_parse +from nebari.plugins import nebari_plugin_manager + + +@pytest.fixture +def qhub_users_import_json(): + return ( + ( + Path(__file__).parent + / "./qhub-config-yaml-files-for-upgrade/qhub-users-import.json" + ) + .read_text() + .rstrip() + ) + + +@pytest.mark.parametrize( + "old_qhub_config_path_str,attempt_fixes,expect_upgrade_error", + [ + ( + "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml", + False, + False, + ), + ( + "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", + False, + True, + ), + ( + "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", + True, + False, + ), + ], +) +def test_upgrade_4_0( + old_qhub_config_path_str, + attempt_fixes, + expect_upgrade_error, + tmp_path, + qhub_users_import_json, + monkeypatch, +): + + def mock_input(prompt, **kwargs): + # Mock different upgrade steps prompt answers + if ( + prompt + == "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] " + ): + return "y" + elif ( + prompt + == "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually." + ): + return "N" + elif ( + prompt + == "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?" + ): + return "y" + elif ( + prompt + == "[bold]Would you like Nebari to assign the corresponding role to all of your current groups automatically?[/bold]" + ): + return "N" + # All other prompts will be answered with "y" + else: + return "y" + + monkeypatch.setattr(Prompt, "ask", mock_input) + + old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str + + tmp_qhub_config = Path(tmp_path, old_qhub_config_path.name) + tmp_qhub_config.write_text(old_qhub_config_path.read_text()) # Copy contents to tmp + + orig_contents = tmp_qhub_config.read_text() # Read in initial contents + + assert not Path(tmp_path, "qhub-users-import.json").exists() + + # Do the upgrade + if not expect_upgrade_error: + do_upgrade( + tmp_qhub_config, attempt_fixes + ) # Would raise an error if invalid by current Nebari version's standards + else: + with pytest.raises(ValueError): + do_upgrade(tmp_qhub_config, attempt_fixes) + return + + # Check the resulting YAML + config = nebari_plugin_manager.read_config(tmp_qhub_config) + + assert len(config.security.keycloak.initial_root_password) == 16 + assert not hasattr(config.security, "users") + assert not hasattr(config.security, "groups") + + __rounded_version__ = rounded_ver_parse(__version__) + + # Check image versions have been bumped up + assert ( + config.default_images.jupyterhub + == f"quansight/nebari-jupyterhub:v{__rounded_version__}" + ) + assert ( + config.profiles.jupyterlab[0].kubespawner_override.image + == f"quansight/nebari-jupyterlab:v{__rounded_version__}" + ) + assert config.security.authentication.type != "custom" + + # Keycloak import users json + assert ( + Path(tmp_path, "nebari-users-import.json").read_text().rstrip() + == qhub_users_import_json + ) + + # Check backup + tmp_qhub_config_backup = Path(tmp_path, f"{old_qhub_config_path.name}.old.backup") + + assert orig_contents == tmp_qhub_config_backup.read_text() + + +@pytest.mark.parametrize( + "version_str, exception", + [ + ("1.0.0", nullcontext()), + ("1.cool.0", pytest.raises(ValueError, match=r"Invalid version string .*")), + ("0,1.0", pytest.raises(ValueError, match=r"Invalid version string .*")), + ("", pytest.raises(ValueError, match=r"Invalid version string .*")), + ( + "1.0.0-rc1", + pytest.raises( + AssertionError, + match=r"Invalid version .*: must be a full release version, not a dev/prerelease/postrelease version", + ), + ), + ( + "1.0.0dev1", + pytest.raises( + AssertionError, + match=r"Invalid version .*: must be a full release version, not a dev/prerelease/postrelease version", + ), + ), + ], +) +def test_version_string(new_upgrade_cls, version_str, exception): + with exception: + + class DummyUpgrade(new_upgrade_cls): + version = version_str + + +def test_duplicated_version(new_upgrade_cls): + duplicated_version = "1.2.3" + with pytest.raises( + AssertionError, match=rf"Duplicate UpgradeStep version {duplicated_version}" + ): + + class DummyUpgrade(new_upgrade_cls): + version = duplicated_version + + class DummyUpgrade2(new_upgrade_cls): + version = duplicated_version + + class DummyUpgrade3(new_upgrade_cls): + version = "1.2.4" From 2c1608a104bfa0255b3d2a30e0fe0ef0c0ecd7b9 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:55:39 +0100 Subject: [PATCH 6/6] readd upgrade tests --- ...ml => qhub-config-aws-310-customauth.yaml} | 31 ++++++++++++------- ...g-do-310.yaml => qhub-config-aws-310.yaml} | 31 ++++++++++++------- tests/tests_unit/test_upgrade.py | 6 ++-- 3 files changed, 43 insertions(+), 25 deletions(-) rename tests/tests_unit/qhub-config-yaml-files-for-upgrade/{qhub-config-do-310-customauth.yaml => qhub-config-aws-310-customauth.yaml} (87%) rename tests/tests_unit/qhub-config-yaml-files-for-upgrade/{qhub-config-do-310.yaml => qhub-config-aws-310.yaml} (86%) diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml similarity index 87% rename from tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml rename to tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml index 50a2b89af4..f68e36a002 100644 --- a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml +++ b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml @@ -1,6 +1,6 @@ -project_name: do-pytest -provider: do -domain: do.nebari.dev +project_name: aws-pytest +provider: aws +domain: aws.nebari.dev certificate: type: self-signed security: @@ -48,22 +48,31 @@ theme: terraform_state: type: remote namespace: dev -digital_ocean: - region: nyc3 - kubernetes_version: 1.21.5-do.0 +amazon_web_services: + kubernetes_version: '1.20' + region: us-east-1 node_groups: general: - instance: s-2vcpu-4gb + instance: m5.2xlarge min_nodes: 1 max_nodes: 1 + gpu: false + single_subnet: false + permissions_boundary: user: - instance: g-2vcpu-8gb - min_nodes: 1 + instance: m5.xlarge + min_nodes: 0 max_nodes: 5 + gpu: false + single_subnet: false + permissions_boundary: worker: - instance: g-2vcpu-8gb - min_nodes: 1 + instance: m5.xlarge + min_nodes: 0 max_nodes: 5 + gpu: false + single_subnet: false + permissions_boundary: profiles: jupyterlab: - display_name: Small Instance diff --git a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310.yaml similarity index 86% rename from tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml rename to tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310.yaml index a3a06da6a2..69eca528a1 100644 --- a/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml +++ b/tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-aws-310.yaml @@ -1,6 +1,6 @@ -project_name: do-pytest -provider: do -domain: do.nebari.dev +project_name: aws-pytest +provider: aws +domain: aws.nebari.dev certificate: type: self-signed security: @@ -45,22 +45,31 @@ theme: terraform_state: type: remote namespace: dev -digital_ocean: - region: nyc3 - kubernetes_version: 1.21.5-do.0 +amazon_web_services: + kubernetes_version: '1.20' + region: us-east-1 node_groups: general: - instance: s-2vcpu-4gb + instance: m5.2xlarge min_nodes: 1 max_nodes: 1 + gpu: false + single_subnet: false + permissions_boundary: user: - instance: g-2vcpu-8gb - min_nodes: 1 + instance: m5.xlarge + min_nodes: 0 max_nodes: 5 + gpu: false + single_subnet: false + permissions_boundary: worker: - instance: g-2vcpu-8gb - min_nodes: 1 + instance: m5.xlarge + min_nodes: 0 max_nodes: 5 + gpu: false + single_subnet: false + permissions_boundary: profiles: jupyterlab: - display_name: Small Instance diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index f6e3f80348..fd70f367f7 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -25,17 +25,17 @@ def qhub_users_import_json(): "old_qhub_config_path_str,attempt_fixes,expect_upgrade_error", [ ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml", + "./qhub-config-yaml-files-for-upgrade/qhub-config-aws-310.yaml", False, False, ), ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", + "./qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml", False, True, ), ( - "./qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml", + "./qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml", True, False, ),