diff --git a/.gitignore b/.gitignore index 60d9bca..1edc0d1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ .work .python-version /clouds.yaml* - +tilt-settings.yaml diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 0000000..eb791bc --- /dev/null +++ b/Tiltfile @@ -0,0 +1,255 @@ +SETTINGS_FILE = "./tilt-settings.yaml" + +# Paths to the required scripts +TILT_IMAGES_APPLY = os.path.abspath("./bin/tilt-images-apply") +TILT_IMAGES_UNAPPLY = os.path.abspath("./bin/tilt-images-unapply") + + +# Allow the use of the azimuth-dev context +allow_k8s_contexts("azimuth-dev") + + +def deep_merge(dict1, dict2): + """ + Deep merges two dictionaries, with values from dict2 taking precedence. + """ + merged = dict(dict1) + for key, value2 in dict2.items(): + if key in dict1: + value1 = dict1[key] + if type(value1) == "dict" and type(value2) == "dict": + merged[key] = deep_merge(value1, value2) + else: + merged[key] = value2 + else: + merged[key] = value2 + return merged + + +# The Tilt settings file is required +if not os.path.exists(SETTINGS_FILE): + fail("settings file must exist at %s" % SETTINGS_FILE) + + +# Load the settings +settings = deep_merge( + { + # The components that will be managed by Tilt, if locally available + # By default, we search for local checkouts as siblings of this checkout + "components": { + "azimuth": { + # Indicates whether the component should be enabled or not + # By default, a component is enabled if the corresponding location exists + # "enabled": True, + + # The location where the component is checked out + # The default location is "../", i.e. siblings of azimuth-config + # "location": "/path/to/component", + + # The name of the Helm release for the component + # Defaults to the component name + # "release_name": "azimuth", + + # The namespace of the Helm release for the component + "release_namespace": "azimuth", + }, + "azimuth-caas-operator": { + "release_namespace": "azimuth", + }, + "azimuth-capi-operator": { + "release_namespace": "azimuth", + }, + "azimuth-identity-operator": { + "release_namespace": "azimuth", + }, + "cluster-api-addon-provider": { + "release_namespace": "capi-addon-system", + }, + "cluster-api-janitor-openstack": { + "release_namespace": "capi-janitor-system", + }, + "zenith": { + "release_name": "zenith-server", + "release_namespace": "azimuth", + }, + }, + }, + read_yaml(SETTINGS_FILE) +) + + +# The user must define an image prefix +if "image_prefix" not in settings: + fail("image_prefix must be specified in %s" % SETTINGS_FILE) + + +def image_name(name): + """ + Returns the full image name with the prefix. + """ + prefix = settings["image_prefix"].removesuffix("/") + return "/".join([prefix, name]) + + +def build_image(name, context, build_args = None): + """ + Defines an image build and returns the image name. + """ + image = image_name(name) + # The Azimuth CaaS operator relies on the .git folder to be in the Docker build context + # This is because it uses pbr for versioning + # Unfortunately, Tilt's docker_build function _always_ ignores the .git directory :-( + # So we use a custom build command + build_command = [ + "docker", + "build", + "-t", + "$EXPECTED_REF", + "--platform", + "linux/amd64", + context, + ] + if build_args: + for arg_name, arg_value in build_args.items(): + build_command.extend([ + "--build-arg", + "'%s=%s'" % (arg_name, arg_value), + ]) + custom_build(image, " ".join(build_command), [context]) + return image + + +def mirror_image(name, source_image): + """ + Defines a mirrored image and returns the image name. + """ + image = image_name(name) + custom_build( + image, + ( + "docker pull --platform linux/amd64 {source_image} && " + + "docker tag {source_image} $EXPECTED_REF" + ).format(source_image = source_image), + [] + ) + return image + + +def port_forward(name, namespace, kind, port): + """ + Runs a port forward as a local resource. + + Could maybe be changed when https://github.com/tilt-dev/tilt/issues/5944 is addressed. + """ + local_resource( + "port-fwd-%s-%s-%s" % (namespace, kind, name), + serve_cmd = "\n".join([ + "while true; do", + " ".join([ + "kubectl", + "port-forward", + "--namespace", + namespace, + "%s/%s" % (kind, name), + port, + ]), + "sleep 1", + "done", + ]) + ) + + +def load_component(name, spec): + """ + Loads a component from the spec. + """ + # If the component is not enabled, we are done + if not spec.get("enabled", True): + print("[%s] component is disabled" % name) + return + + # By default, we search for local checkouts as siblings of this checkout + location = spec.get("location", "../%s" % name) + # If the location does not exist, we are done + if not os.path.exists(location): + print("[%s] location '%s' does not exist - ignoring" % (name, location)) + return + + # Next, read the component file if present + component_file = os.path.join(location, "tilt-component.yaml") + component_spec = read_yaml(component_file, default = None) or {} + + # Define a docker build resource for each image, storing the paths as we go + images = [] + image_paths = [] + if "images" in component_spec: + # If there are images defined in the spec, use those + for image_name, image_spec in component_spec["images"].items(): + if image_spec.get("action", "build") == "build": + image = build_image( + image_name, + os.path.join(location, image_spec["context"]), + image_spec.get("build_args", {}) + ) + else: + image = mirror_image(image_name, image_spec["source_image"]) + images.append(image) + # By default, assume that images are set with a top-level 'image' variable + image_paths.append(image_spec.get("chart_path", "image")) + elif os.path.exists(os.path.join(location, "Dockerfile")): + # If a Dockerfile exists at the top level, assume it is the only image and + # that it is set in the chart with a top-level 'image' variable + images.append(build_image(name, location)) + image_paths.append("image") + + # Get the chart path + # We assume the chart is at './chart', but allow the component to override + chart_path_rel = component_spec.get("chart", "./chart") + chart_path = os.path.join(location, chart_path_rel) + + # Define a custom deploy to replace the images in an existing Helm release + env = { + "TILT_RELEASE_NAME": spec.get("release_name", name), + "TILT_RELEASE_NAMESPACE": spec["release_namespace"], + "TILT_CHART_PATH": chart_path, + } + for i, image_path in enumerate(image_paths): + env["TILT_IMAGE_PATH_%s" % i] = image_path + + k8s_custom_deploy( + name, + apply_cmd = TILT_IMAGES_APPLY, + apply_env = env, + delete_cmd = TILT_IMAGES_UNAPPLY, + delete_env = env, + # Don't include the lock and subcharts + deps = [ + os.path.join(chart_path, ".helmignore"), + os.path.join(chart_path, "Chart.yaml"), + os.path.join(chart_path, "values.yaml"), + os.path.join(chart_path, "crds"), + os.path.join(chart_path, "files"), + os.path.join(chart_path, "templates"), + ], + image_deps = images + ) + + # Set up any port forwards for the component + for pfwd_spec in component_spec.get("port_forwards", []): + port_forward( + pfwd_spec["name"], + spec["release_namespace"], + pfwd_spec["kind"], + pfwd_spec["port"] + ) + + # Create any local resources for the component + for name, lr_spec in component_spec.get("local_resources", {}).items(): + lr_spec.setdefault("dir", location) + lr_spec.setdefault("serve_dir", location) + local_resource(name, **lr_spec) + + +# Load the components defined in the settings +for name, spec in settings["components"].items(): + load_component(name, spec) diff --git a/bin/tilt-images-apply b/bin/tilt-images-apply new file mode 100755 index 0000000..9a71625 --- /dev/null +++ b/bin/tilt-images-apply @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +import contextlib +import json +import os +import pathlib +import re +import subprocess +import sys + +import yaml + + +TILT_IMAGE_REGEX = r"^TILT_IMAGE_(\d+)$" + + +def exec_cmd(cmd, **kwargs): + # Log the command we are about to execute + print("Executing cmd: {}".format(" ".join(cmd)), file = sys.stderr) + # By default, redirect stdout to stderr and raise on error + kwargs.setdefault("stdout", sys.stderr) + kwargs.setdefault("check", True) + # Run the command and return the process instance + return subprocess.run(cmd, **kwargs) + + +# Before doing anything, check if we need to record the revision +# This will be used to rollback on tilt down to the last "normal" version +revision_file = pathlib.Path(os.environ["AZIMUTH_TILT_WORK_DIR"]) / "{}.{}.rev".format( + os.environ["TILT_RELEASE_NAME"], + os.environ["TILT_RELEASE_NAMESPACE"] +) +if not revision_file.exists(): + revision_file.touch() + fh = revision_file.open("w") + try: + status_result = exec_cmd( + [ + "helm", + "status", + os.environ["TILT_RELEASE_NAME"], + "--namespace", + os.environ["TILT_RELEASE_NAMESPACE"], + "--output", + "json", + ], + stdout = subprocess.PIPE + ) + revision = json.loads(status_result.stdout)["version"] + fh.write(str(revision)) + except Exception as exc: + # If there was an error writing, unlink the file after closing + fh.close() + revision_file.unlink() + raise + else: + fh.close() + + +# Update the dependencies of the chart +exec_cmd(["helm", "dependency", "update", os.environ["TILT_CHART_PATH"]]) + + +# Build and run the Helm upgrade command +# We reuse the values from the previous installation, but overwrite any images +# specified in the Tiltfile +helm_upgrade_command = [ + "helm", + "upgrade", + os.environ["TILT_RELEASE_NAME"], + os.environ["TILT_CHART_PATH"], + "--namespace", + os.environ["TILT_RELEASE_NAMESPACE"], + "--reuse-values", +] + +idx = 0 +while True: + try: + image = os.environ[f"TILT_IMAGE_{idx}"] + except KeyError: + break + else: + repository, tag = image.rsplit(":", maxsplit = 1) + path = os.environ[f"TILT_IMAGE_PATH_{idx}"] + helm_upgrade_command.extend([ + "--set", + f"{path}.repository={repository}", + "--set", + f"{path}.tag={tag}", + ]) + idx = idx + 1 + +exec_cmd(helm_upgrade_command) + + +# Finally, print the currently installed manifest so Tilt knows about the resources +manifest = exec_cmd( + [ + "helm", + "get", + "manifest", + os.environ["TILT_RELEASE_NAME"], + "--namespace", + os.environ["TILT_RELEASE_NAMESPACE"] + ], + stdout = subprocess.PIPE +) + +# Tilt wants UIDs, so we have to process the manifest to add the namespace where +# required before passing through kubectl +manifest_objs = list(yaml.safe_load_all(manifest.stdout)) +for obj in manifest_objs: + obj["metadata"].setdefault("namespace", os.environ["TILT_RELEASE_NAMESPACE"]) +exec_cmd( + [ + "kubectl", + "get", + "--filename", + "-", + "--output", + "yaml", + ], + input = yaml.safe_dump_all(manifest_objs).encode(), + stdout = None +) diff --git a/bin/tilt-images-unapply b/bin/tilt-images-unapply new file mode 100755 index 0000000..77433a7 --- /dev/null +++ b/bin/tilt-images-unapply @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Roll back to the revision specified in the file +revision_file="$AZIMUTH_TILT_WORK_DIR/$TILT_RELEASE_NAME.$TILT_RELEASE_NAMESPACE.rev" +if [ -f "$revision_file" ]; then + revision="$(cat "$revision_file")" + echo "Executing cmd: helm rollback $TILT_RELEASE_NAME $revision --namespace $TILT_RELEASE_NAMESPACE" + helm rollback "$TILT_RELEASE_NAME" "$revision" --namespace "$TILT_RELEASE_NAMESPACE" + rm "$revision_file" +fi diff --git a/bin/tilt-up b/bin/tilt-up new file mode 100755 index 0000000..3f805d6 --- /dev/null +++ b/bin/tilt-up @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +##### +## This script uses Tilt (tilt.dev) to allow easier code development on the +## currently activated environment +##### + +set -eo pipefail + + +if [ -z "$AZIMUTH_CONFIG_ROOT" ] || [ -z "$AZIMUTH_CONFIG_ENVIRONMENT_ROOT" ]; then + echo "Please activate an environment" >&2 + exit 1 +fi + + +# Check that the Tilt settings file exists +if [ ! -f "$AZIMUTH_CONFIG_ROOT/tilt-settings.yaml" ]; then + echo "[ERROR] Could not find tilt-settings.yaml" 1>&2 + exit 1 +fi + + +ansible_variable() { + ANSIBLE_LOAD_CALLBACK_PLUGINS=true \ + ANSIBLE_STDOUT_CALLBACK=json \ + ansible -m debug -a "var=$1" all | \ + jq -r ".plays[0].tasks[0].hosts.localhost.$1" +} + + +# Variables to hold the PIDs of the SSH connection and tilt +ssh_pid= +tilt_pid= +work_dir= + +# Function to terminate background processes when the script exits +terminate_bg_pids() { + set +e + # Make sure tilt up is dead + test -n "$tilt_pid" && kill -0 "$tilt_pid" >/dev/null 2>&1 && kill "$tilt_pid" + # Run tilt down + tilt down + # Kill the SSH tunnel + test -n "$ssh_pid" && kill -0 "$ssh_pid" >/dev/null 2>&1 && kill "$ssh_pid" + # Clean up the working directory + test -n "$work_dir" && rm -rf "$work_dir" +} +trap 'terminate_bg_pids' EXIT + +# The SOCKS port is the only input +socks_port="${1:-1080}" + +# Make a working directory for tilt related stuff +echo "Creating working directory..." +work_dir="$(ansible_variable work_directory)/tilt" +mkdir -p "$work_dir" +KUBECONFIG="$work_dir/kubeconfig" + + +echo "Fetching kubeconfig for Azimuth Kubernetes cluster..." +install_mode="$(ansible_variable install_mode)" +if [ "$install_mode" = "ha" ]; then + cluster_name="$(ansible_variable capi_cluster_release_name)" + kubeconfig_arg="KUBECONFIG=./kubeconfig-${cluster_name}.yaml" +fi +"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" \ + $kubeconfig_arg \ + kubectl config view --raw > "$KUBECONFIG" + + +echo "Updating kubeconfig with SOCKS proxy..." +export KUBECONFIG +ctx="$(kubectl config current-context)" +cluster="$(kubectl config view -o jsonpath="{.contexts[?(@.name == \"$ctx\")].context.cluster}")" +kubectl config set-cluster $cluster --proxy-url="socks5://localhost:$socks_port" + +echo "Renaming context to azimuth-dev..." +kubectl config rename-context $ctx azimuth-dev + + +echo "Starting SOCKS proxy..." +"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" -D $socks_port -N & +ssh_pid="$!" +# Wait a few seconds and check that the process is running +sleep 5 +if ! kill -0 "$ssh_pid" >/dev/null 2>&1; then + echo "[ERROR] Failed to connect to Azimuth cluster" 1>&2 + exit 1 +fi + +# Use the working directory as TMP for tilt +export AZIMUTH_TILT_WORK_DIR="$work_dir" + +echo "Running 'tilt up'..." +tilt up & +tilt_pid="$!" +# Spin until one of the processes exits +wait $ssh_pid $tilt_pid diff --git a/docs/developing/index.md b/docs/developing/index.md new file mode 100644 index 0000000..efcc021 --- /dev/null +++ b/docs/developing/index.md @@ -0,0 +1,194 @@ +# Developing Azimuth + +An Azimuth deployment consists of several interdependent components, some of which are +Azimuth-specific and some of which are third-party components. Plugging code under +development into such a system can be tricky, making development difficult and slow. + +## Developing Azimuth components + +Azimuth has a number of components, mostly written in Python: + + * [Azimuth API and UI](https://github.com/stackhpc/azimuth) - user-facing API and UI + * [Azimuth CaaS operator](https://github.com/stackhpc/azimuth-caas-operator) - Kubernetes operator implementing CaaS functionality + * [Azimuth CAPI operator](https://github.com/stackhpc/azimuth-capi-operator) - Kubernetes operator implementing Kubernetes and Kubernetes App functionality + * [Azimuth identity operator](https://github.com/stackhpc/azimuth-identity-operator) - Kubernetes operator implementing platform identity + * [Zenith](https://github.com/stackhpc/zenith) - secure, tunnelling application proxy used to expose platform services + * [Cluster API addon provider](https://github.com/stackhpc/cluster-api-addon-provider) - addons for Cluster API clusters + * [Cluster API janitor for OpenStack](https://github.com/stackhpc/cluster-api-janitor-openstack) - resource cleanup for Cluster API clusters on OpenStack clouds + +It is useful to develop these components in the context of a running Azimuth installation, +as they have dependencies on each other. + +To enable this, Azimuth uses [Tilt](https://tilt.dev/) to provide a developer environment +where code under development is automatically built and injected into a live system that +you can interact with. Tilt provides a dashboard that can be used to drill down into +build failures and the logs of the components under development. + +![Azimuth in Tilt](tilt-interface.png) + +### Prerequisites + +In order to use Tilt to develop Azimuth, the following tools must be available on your +development machine (in addition to those required to install Azimuth itself): + + * The [Tilt CLI](https://docs.tilt.dev/install.html) + * A `docker` command, e.g. [Docker Desktop](https://docs.docker.com/desktop/) + * The [kubectl command](https://kubernetes.io/docs/tasks/tools/#kubectl) + * The [Helm CLI](https://helm.sh/docs/intro/install/) + +For developing the Azimuth UI, the following are also required: + + * [node.js](https://nodejs.org) + * The [Yarn Classic](https://classic.yarnpkg.com/lang/en/docs/install/) package manager + +### Deploying a dev instance + +To use Tilt for developing Azimuth components, you first need a running Azimuth instance. + +Each developer should have their own independent instance of Azimuth as Tilt will make +changes to the running Azimuth components, based on the code under development, that may +disrupt or break things for others. + +!!! tip + + A single node deployment, e.g. a [demo deployment](../try.md), is sufficient for developing + the Azimuth components. + + You may wish to maintain a [development environment](../environments.md) containing + site-specific customisations. + +The following instructions assume that your Azimuth configuration contains a developer environment +called `dev`. It is assumed that you have your Azimuth configuration checked out and that you have +an [application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html) +for the target cloud. + +To stand up your developer-specific Azimuth instance, using the `dev` environment, use the +following: + +```bash +# Set OpenStack configuration variables +export OS_CLOUD=openstack +export OS_CLIENT_CONFIG_FILE=/path/to/clouds.yaml + +# Activate the dev config environment with a specific instance name +# +# This means that resources created for the instance will not collide +# with other deployments that use the dev environment +source ./bin/activate dev jbloggs-dev + +# Install Azimuth as usual +ansible-galaxy install -f -r requirements.yml +ansible-playbook stackhpc.azimuth_ops.provision +``` + +### Configuring a container registry + +Azimuth's Tilt configuration looks for a file called `tilt-settings.yaml` that defines settings +for the development environment. This file is specific to you and should not be added to version +control (it is specified in `.gitignore`). + +In order to get the code under development into your running Azimuth instance, Tilt must have +access to a container registry that is accessible to both your development machine and the +Azimuth instance. In response to code changes, Tilt will automatically build and push images +to this registry and then configure the Azimuth instance to use them. + +To configure the prefix for images built by Tilt, use the following setting: + +```yaml title="tilt-settings.yaml" +# Images will be pushed to: +#  ghcr.io/jbloggs/azimuth-api +#  ghcr.io/jbloggs/azimuth-ui +#  ghcr.io/jbloggs/azimuth-caas-operator +# ... +image_prefix: ghcr.io/jbloggs +``` + +!!! tip + + A good candidate for this is to use [GitHub Packages](https://github.com/features/packages) + with your user account, as in the example above. This means that development builds do not + require access to or clutter up the production repositories. + + When using GitHub Packages, the repositories that are created by Tilt when it builds images + for the first time will be private. You must log into GitHub and make them public before + your Azimuth instance can use them. Until you do this, you will see image pull errors in + the Tilt interface. + +### Using the Tilt environment + +Tilt will look for checkouts of Azimuth components as siblings of your Azimuth configuration +and include them in your development environment. For example, the following directory structure +will result in a development environment where changes to the Azimuth API, UI and CaaS operators +are built and pushed into your development Azimuth instance for testing: + +``` +. +├── azimuth +│ ├── api +│ ├── chart +│ ├── ui +│ └── ... +├── azimuth-caas-operator +│ ├── Dockerfile +│ ├── azimuth_caas_operator +│ ├── charts +│ └── ... +└── myorg-azimuth-config + ├── Tiltfile + ├── bin + ├── environments + ├── tilt-settings.yaml + └── ... +``` + +If you wish to prevent a particular component being included in your development environment, +even if the checkout exists as a sibling, you can configure this using the `enabled` flag for +the component in `tilt-settings.yaml`. For example, the following would prevent the CaaS +operator from being included in the development environment, even with the directory structure +above: + +```yaml title="tilt-settings.yaml" +components: + azimuth-caas-operator: + enabled: false +``` + +Once you have checked out all of the components that you want to develop, you can start the +development environment using: + +```bash +./bin/tilt-up +``` + +This will configure Tilt to connect to your Azimuth instance and begin watching your local +checkouts for changes. When a change is detected, Tilt will build and push an image for the +component before reconfiguring the Helm release for the component to point to the new image. +Once this process is complete, you can interact with your changes using the Azimuth UI for +your instance. + +!!! tip + + Press the space bar to launch the [Tilt user interface](https://docs.tilt.dev/tutorial/3-tilt-ui.html). + +When the `tilt-up` command is terminated, all of the Helm releases for the components that +were under development are rolled back to the version that was running before the command +was started. + +### Local UI development + +Because of how the user interface is optimised for production, the container image build for +the Azimuth UI is very slow even for a minor change. Because of this, the usual Tilt flow of +build/push/deploy is not suitable for UI development. + +To improve the feedback cycle for UI development, the Azimuth Tilt environment also builds +and runs the Azimuth UI locally using the +[webpack DevServer](https://webpack.js.org/configuration/dev-server/). The UI communicates +with the Azimuth API on your Azimuth dev instance using a forwarded port (this is necessary +in order for the cookie-based authentication to work properly). + +The local version of the UI is available at `http://localhost:3000`. + +!!! note + + The UI container image is still built, pushed and deployed in the background. + However changes made to the JS files will be visible in the local version much faster. diff --git a/docs/developing/tilt-interface.png b/docs/developing/tilt-interface.png new file mode 100644 index 0000000..46434ca Binary files /dev/null and b/docs/developing/tilt-interface.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 6ee6b34..d00e23f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,6 +40,8 @@ nav: - debugging/consul.md - debugging/kubernetes.md - debugging/caas.md + - Developing: + - developing/index.md theme: name: material