diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index ab26f0c..3d3f0c1 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -71,7 +71,7 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@1.1.0 with: - juju-channel: "3.1/stable" + juju-channel: 3.1/stable provider: microk8s channel: 1.28-strict/stable microk8s-addons: "dns storage rbac metallb:10.64.140.43-10.64.140.49" @@ -88,6 +88,10 @@ jobs: run: juju status if: failure() + - name: Get juju debug logs + run: juju debug-log --limit 100 + if: failure() + - name: Get workload logs run: kubectl logs --tail 100 -ntesting -lapp.kubernetes.io/name=zenml-server if: failure() diff --git a/README.md b/README.md index 1181c58..ba4a615 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,18 @@ **DISCLAIMER:** This project was inspired by the [Charmed MLFlow Project](https://github.com/canonical/mlflow-operator) and also implements solutions found there. -- [ZenML on Juju with Microk8s](#zenml-on-juju-with-microk8s) +- [Get Started](#get-started) - - [Get Started](#get-started) + - [Prerequisites](#prerequisites) + - [Install and prepare Juju](#install-and-prepare-juju) + - [Deploy Sandalone ZenML Server Bundle](#build-and-deploy-the-charm-manually) + - [Build and deploy the charm manually](#build-and-deploy-the-charm-manually) - - [Prerequisites](#prerequisites) - - [Install and prepare Juju](#install-and-prepare-juju) - - [Deploy Sandalone ZenML Server Bundle](#build-and-deploy-the-charm-manually) - - [Build and deploy the charm manually](#build-and-deploy-the-charm-manually) +- [Integrate ZenML Server with Charmed Kubeflow](#integrate-zenml-server-with-charmed-kubeflow) - - [Integrate ZenML Server with Charmed Kubeflow](#integrate-zenml-server-with-charmed-kubeflow) +- [Ingress](#ingress) - - [Examples](#examples) +- [Examples](#examples) ## Get started @@ -261,6 +261,42 @@ zenml container-registry register \ --uri=localhost:32000 # or for EC2 :32000 ``` +## Ingress + +Currently the charmed `zenml-server-operator` supports ingress integration with [`Charmed Istio`](https://github.com/canonical/istio-operators) + +**NOTE:** According to [ZenML Docs](https://docs.zenml.io/deploying-zenml/zenml-self-hosted/deploy-with-helm#use-a-dedicated-ingress-url-path-for-zenml): + +``` +This method has one current limitation: the ZenML UI does not support URL rewriting and will not work properly if you use a dedicated Ingress URL path. You can still connect your client to the ZenML server and use it to run pipelines as usual, but you will not be able to use the ZenML UI. +``` + +**Additionally, the ZenML client does not support DEX auth - this might require configuring the server to use DEX as an external auth provider - more info [here](https://github.com/zenml-io/zenml/blob/main/src/zenml/zen_server/deploy/helm/values.yaml)** + +Deploy Istio Gateway and Istio Pilot charms and configure the relation + +```bash +juju deploy istio-gateway istio-ingressgateway --channel 1.17/stable --config kind=ingress --trust + +juju deploy istio-pilot --channel 1.17/stable --config default-gateway=test-gateway -trust + +juju relate istio-pilot istio-ingressgateway +``` + +To integrate `zenml-server` with currently deployed `istio-operator` run the command: + +```bash +juju relate zenml-server istio-pilot +``` + +After this done the ZenML Client can connect to the server over the ingress IP found by running: + +```bash +microk8s kubectl -n get svc istio-ingressgateway-workload -o jsonpath='{.status.loadBalancer.ingress[0].ip}' +``` + +Over the URL `http://zenml/` + ## Examples Check out the [examples directory](/examples/) diff --git a/metadata.yaml b/metadata.yaml index 350f13f..78c5817 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -15,3 +15,43 @@ resources: requires: relational-db: interface: mysql_client + ingress: + interface: ingress + schema: + v2: + requires: + type: object + properties: + service: + type: string + port: + type: integer + namespace: + type: string + prefix: + type: string + rewrite: + type: string + required: + - service + - port + - namespace + - prefix + v1: + requires: + type: object + properties: + service: + type: string + port: + type: integer + prefix: + type: string + rewrite: + type: string + required: + - service + - port + - prefix + versions: [v1] + __schema_source: https://raw.githubusercontent.com/canonical/operator-schemas/master/ingress.yaml diff --git a/requirements-integration.txt b/requirements-integration.txt index f07e493..c38508f 100644 --- a/requirements-integration.txt +++ b/requirements-integration.txt @@ -1,6 +1,5 @@ aiohttp jinja2 -# Pinning to <4.0 due to compatibility with the 3.1 controller version juju<4.0 pytest-operator requests diff --git a/src/charm.py b/src/charm.py index 3d6e956..41ac260 100755 --- a/src/charm.py +++ b/src/charm.py @@ -312,13 +312,29 @@ def _on_database_created(self, event) -> None: self._on_event(event) + def _send_ingress_info(self, interfaces): + if interfaces["ingress"]: + interfaces["ingress"].send_data( + { + "prefix": "/zenml/", + "rewrite": "/", + "service": self.model.app.name, + "namespace": self.model.name, + "port": int(self._port), + } + ) + def _on_event(self, event) -> None: """Perform all required actions for the Charm.""" try: self._check_leader() + interfaces = self._get_interfaces() relational_db_data = self._get_relational_db_data() envs = self._get_env_vars(relational_db_data) + if interfaces.get("ingress"): + envs["ZENML_SERVER_ROOT_URL_PATH"] = "/zenml" + if not self.container.can_connect(): raise ErrorWithStatus( f"Container {self._container_name} is not ready", WaitingStatus @@ -326,6 +342,7 @@ def _on_event(self, event) -> None: self._update_layer( self.container, self._container_name, self._charmed_zenml_layer(envs) ) + self._send_ingress_info(interfaces) except ErrorWithStatus as err: self.model.unit.status = err.status self.logger.info(f"Event {event} stopped early with message: {str(err)}") diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 6a70d1b..33bcd0b 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,16 +4,64 @@ import time from pathlib import Path +import lightkube import pytest import yaml +from lightkube.resources.core_v1 import Service from pytest_operator.plugin import OpsTest -from tenacity import retry, stop_after_attempt +from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed logger = logging.getLogger(__name__) METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) CHARM_NAME = METADATA["name"] RELATIONAL_DB_CHARM_NAME = "mysql-k8s" +ISTIO_GATEWAY_CHARM_NAME = "istio-ingressgateway" +ISTIO_PILOT_CHARM_NAME = "istio-pilot" + + +@pytest.fixture(scope="session") +def lightkube_client() -> lightkube.Client: + client = lightkube.Client(field_manager=CHARM_NAME) + return client + + +async def setup_istio(ops_test: OpsTest, istio_gateway: str, istio_pilot: str): + """Deploy Istio Ingress Gateway and Istio Pilot.""" + await ops_test.model.deploy( + entity_url="istio-gateway", + application_name=istio_gateway, + channel="1.17/stable", + config={"kind": "ingress"}, + trust=True, + ) + await ops_test.model.deploy( + istio_pilot, + channel="1.17/stable", + config={"default-gateway": "test-gateway"}, + trust=True, + ) + await ops_test.model.add_relation(istio_pilot, istio_gateway) + + await ops_test.model.wait_for_idle( + apps=[istio_pilot, istio_gateway], + status="active", + timeout=60 * 5, + raise_on_blocked=False, + raise_on_error=False, + ) + + +def get_ingress_url(lightkube_client: lightkube.Client, model_name: str): + gateway_svc = lightkube_client.get( + Service, "istio-ingressgateway-workload", namespace=model_name + ) + ingress_record = gateway_svc.status.loadBalancer.ingress[0] + if ingress_record.ip: + public_url = f"http://{ingress_record.ip}.nip.io" + if ingress_record.hostname: + public_url = f"http://{ingress_record.hostname}" # Use hostname (e.g. EKS) + return public_url class TestCharm: @@ -75,3 +123,27 @@ async def test_successfull_deploy_senario(self, ops_test: OpsTest): if zenml_subprocess.stderr: logger.info(f"ZenML command stderr: {zenml_subprocess.stderr}") assert zenml_subprocess.returncode == 0 + + async def test_ingress_relation(self, ops_test: OpsTest): + """Setup Istio and relate it to the MLflow.""" + await setup_istio(ops_test, ISTIO_GATEWAY_CHARM_NAME, ISTIO_PILOT_CHARM_NAME) + + await ops_test.model.add_relation( + f"{ISTIO_PILOT_CHARM_NAME}:ingress", f"{CHARM_NAME}:ingress" + ) + + await ops_test.model.wait_for_idle(apps=[CHARM_NAME], status="active", timeout=60 * 5) + + @retry(stop=stop_after_delay(600), wait=wait_fixed(10)) + @pytest.mark.abort_on_fail + async def test_ingress_url(self, lightkube_client, ops_test: OpsTest): + ingress_url = get_ingress_url(lightkube_client, ops_test.model_name) + + zenml_url = f"{ingress_url}/zenml/" + zenml_subprocess = subprocess.run( + ["zenml", "connect", "--url", zenml_url, "--username", "default", "--password", ""] + ) + logger.info(f"ZenML command stdout: {zenml_subprocess.stdout}") + if zenml_subprocess.stderr: + logger.info(f"ZenML command stderr: {zenml_subprocess.stderr}") + assert zenml_subprocess.returncode == 0