Merge pull request #23 from zalando-incubator/k8s-custom-resources

Support K8s custom resources

Author: tortila

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.14][] - 2019-11-29
+
+### Added
+
+  - Support for custom Kubernetes resources (#23)
+
 ## [1.2.13][] - 2019-10-29
 
 ### Fixed
@@ -107,6 +113,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Implemented `-v`/`--version` option to show Zelt version.
   - This changelog.
 
+[1.2.14]: https://github.com/zalando-incubator/zelt/compare/v1.2.13...v1.2.14
 [1.2.13]: https://github.com/zalando-incubator/zelt/compare/v1.2.12...v1.2.13
 [1.2.12]: https://github.com/zalando-incubator/zelt/compare/v1.2.11...v1.2.12
 [1.2.11]: https://github.com/zalando-incubator/zelt/compare/v1.2.10...v1.2.11

docs/Run-a-load-test.rst

@@ -92,6 +92,11 @@ These consist of:
 - ``service.yaml`` that defines the Kubernetes service_ to create
 - ``ingress.yaml`` that defines the Kubernetes ingress_ to create
 
+.. note::
+    Currently Zelt's support is limited to the above Kubernetes resource types, as well as `custom resources`_.
+    Please note that Zelt will deploy a custom resource only if its `custom resource definition`_ is already deployed in the cluster, and only if
+    the scope of the resource is limited to a namespace.
+
 .. TODO: Create a page detailing each manifest
 .. For more detailed information, please refer to :ref:`manifests`.
 
@@ -195,3 +200,5 @@ before doing this or they will be deleted!
 .. _`Download them`: https://github.com/zalando-incubator/zelt/tree/master/examples/manifests/combined
 .. _Transformer: https://github.com/zalando-incubator/Transformer
 .. _`Locust's documentation`: https://docs.locust.io/en/stable/what-is-locust.html
+.. _`custom resources`: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
+.. _`custom resource definition`: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zelt"
-version = "1.2.13"
+version = "1.2.14"
 description = "Zalando end-to-end load tester"
 authors = [
     "Brian Maher <brian.maher@zalando.de>",

tests/kubernetes/test_client.py

@@ -1,3 +1,4 @@
+from typing import List
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -20,6 +21,7 @@
     delete_deployments,
     await_no_resources_found,
     rescale_deployment,
+    try_creating_custom_objects,
 )
 from zelt.kubernetes.manifest import Manifest
 
@@ -360,3 +362,183 @@ def test_it_raises_exception(self, delete, waiting):
 
         delete.assert_called_once()
         waiting.assert_not_called()
+
+
+class TestCreateCustomResources:
+    @pytest.fixture()
+    def manifests(self) -> List[Manifest]:
+        manifest_a = Manifest(
+            body={
+                "apiVersion": "zalando.org/v1",
+                "kind": "ZalandoCustomResource",
+                "metadata": {
+                    "name": "a_surprise",
+                    "namespace": "a_namespace",
+                    "labels": {"application": "an_application"},
+                },
+            }
+        )
+        manifest_b = Manifest(
+            body={
+                "apiVersion": "example.com/v2",
+                "kind": "ExampleCustomResource",
+                "metadata": {
+                    "name": "a_surprise",
+                    "namespace": "a_namespace",
+                    "labels": {"application": "an_application"},
+                },
+            }
+        )
+        manifest_c = Manifest(
+            body={
+                "apiVersion": "example.com/v2",
+                "kind": "ClusterCustomResource",
+                "metadata": {
+                    "name": "a_surprise",
+                    "labels": {"application": "an_application"},
+                },
+            }
+        )
+        return [manifest_a, manifest_b, manifest_c]
+
+    @pytest.fixture()
+    def crds(self) -> List[MagicMock]:
+        crd_manifest_a = MagicMock()
+        crd_manifest_a.spec.names.kind = "ZalandoCustomResource"
+        crd_manifest_a.spec.names.plural = "zalandocustomresources"
+        crd_manifest_a.spec.scope = "Namespaced"
+
+        crd_manifest_b = MagicMock()
+        crd_manifest_b.spec.names.kind = "ExampleCustomResource"
+        crd_manifest_b.spec.names.plural = "examplecustomresources"
+        crd_manifest_b.spec.scope = "Namespaced"
+
+        crd_cluster_manifest = MagicMock()
+        crd_cluster_manifest.spec.names.kind = "ClusterCustomResource"
+        crd_cluster_manifest.spec.names.plural = "clustercustomresources"
+        crd_cluster_manifest.spec.scope = "Cluster"
+
+        return [crd_manifest_a, crd_manifest_b, crd_cluster_manifest]
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi.create_namespaced_custom_object")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_fetches_available_crds_once(
+        self, list_crds, create_custom_objects, config, manifests, crds
+    ):
+        try_creating_custom_objects(manifests)
+        list_crds.assert_called_once()
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_creates_supported_custom_resources(
+        self,
+        list_crds,
+        custom_objects_api,
+        config,
+        manifests: List[Manifest],
+        crds: List[MagicMock],
+    ):
+        # All 3 CRDs are available in the cluster
+        list_crds.return_value = MagicMock(items=crds)
+        # Zelt tries to deploy only the namespaced ones
+        # (without a CustomClusterResource)
+        try_creating_custom_objects(manifests[:2])
+
+        custom_objects_api().create_namespaced_custom_object.assert_any_call(
+            namespace=manifests[0].namespace,
+            body=manifests[0].body,
+            group="zalando.org",
+            version="v1",
+            plural="zalandocustomresources",
+        )
+        custom_objects_api().create_namespaced_custom_object.assert_any_call(
+            namespace=manifests[1].namespace,
+            body=manifests[1].body,
+            group="example.com",
+            version="v2",
+            plural="examplecustomresources",
+        )
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_ignores_unsupported_resources(
+        self, list_crds, custom_objects_api, config, manifests, crds, caplog
+    ):
+        # ZalandoCustomResource is the only available CRD in the cluster
+        list_crds.return_value = MagicMock(items=[crds[0]])
+        # Zelt tries to deploy ZalandoCustomResource and ExampleCustomResource
+        try_creating_custom_objects(manifests)
+
+        custom_objects_api().create_namespaced_custom_object.assert_called_once()
+        custom_objects_api().create_namespaced_custom_object.assert_called_with(
+            namespace=manifests[0].namespace,
+            body=manifests[0].body,
+            group="zalando.org",
+            version="v1",
+            plural="zalandocustomresources",
+        )
+        assert any(
+            "Unsupported custom manifest" in r.msg for r in caplog.records
+        ), "an error should be logged"
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_ignores_non_namespaced_crds(
+        self, list_crds, custom_objects_api, config, manifests, crds, caplog
+    ):
+        # All 3 CRDs are available in the cluster
+        list_crds.return_value = MagicMock(items=crds)
+        # Zelt tries to deploy a ClusterCustomResource
+        try_creating_custom_objects([manifests[2]])
+
+        custom_objects_api().create_namespaced_custom_object.assert_not_called()
+        assert any(
+            "Non-namespaced resources are not supported" in r.msg
+            for r in caplog.records
+        ), "an error should be logged"
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_raises_exception_on_fetching_crds(
+        self, list_crds, custom_objects_api, config, manifests
+    ):
+        list_crds.side_effect = ApiException()
+
+        with pytest.raises(ApiException):
+            try_creating_custom_objects(manifests)
+
+        list_crds.assert_called_once()
+        custom_objects_api().create_namespaced_custom_object.assert_not_called()
+
+    @patch("kubernetes.config.load_kube_config")
+    @patch("zelt.kubernetes.client.CustomObjectsApi")
+    @patch(
+        "zelt.kubernetes.client.ApiextensionsV1beta1Api.list_custom_resource_definition"
+    )
+    def test_it_raises_exception_on_creating_resources(
+        self, list_crds, custom_objects_api, config, manifests, crds
+    ):
+        list_crds.return_value = MagicMock(items=crds)
+        custom_objects_api().create_namespaced_custom_object.side_effect = (
+            ApiException()
+        )
+
+        with pytest.raises(ApiException):
+            try_creating_custom_objects(manifests)
+
+        list_crds.assert_called_once()

tests/kubernetes/test_deployer.py

@@ -37,6 +37,7 @@ def manifest_set(tmp_path: Path) -> ManifestSet:
         ingress=Manifest.from_file(manifest_file),
         controller=Manifest.from_file(manifest_file),
         worker=Manifest.from_file(manifest_file),
+        others=[],
     )
 
 
@@ -55,6 +56,7 @@ def configmap_storage() -> ConfigmapStorage:
 
 class TestCreateResources:
     @patch("zelt.kubernetes.client.config")
+    @patch("zelt.kubernetes.client.try_creating_custom_objects")
     @patch("zelt.kubernetes.client.CoreV1Api.create_namespace")
     @patch("zelt.kubernetes.client.CoreV1Api.create_namespaced_service")
     @patch("zelt.kubernetes.client.CoreV1Api.create_namespaced_config_map")
@@ -69,6 +71,7 @@ def test_it_deploys_all_given_manifests_and_configmap(
         create_configmap,
         create_service,
         create_namespace,
+        create_custom_objects,
         config,
         configmap_storage: ConfigmapStorage,
         locustfile: Path,
@@ -81,8 +84,37 @@ def test_it_deploys_all_given_manifests_and_configmap(
         create_service.assert_called_once()
         create_configmap.assert_called_once()
         create_ingress.assert_called_once()
+        create_custom_objects.assert_not_called()
         assert create_deployment.call_count == 2
 
+    @patch("zelt.kubernetes.client.config")
+    @patch("zelt.kubernetes.client.try_creating_custom_objects")
+    @patch("zelt.kubernetes.client.CoreV1Api.create_namespace")
+    @patch("zelt.kubernetes.client.CoreV1Api.create_namespaced_service")
+    @patch("zelt.kubernetes.client.CoreV1Api.create_namespaced_config_map")
+    @patch("zelt.kubernetes.client.NetworkingV1beta1Api.create_namespaced_ingress")
+    @patch("zelt.kubernetes.client.AppsV1Api.create_namespaced_deployment")
+    @patch("zelt.kubernetes.client.wait_until_pod_ready")
+    def test_it_deploys_custom_manifests(
+        self,
+        wait,
+        create_deployment,
+        create_ingress,
+        create_configmap,
+        create_service,
+        create_namespace,
+        create_custom_objects,
+        config,
+        configmap_storage: ConfigmapStorage,
+        locustfile: Path,
+        manifest_set: ManifestSet,
+    ):
+        manifest_set = manifest_set._replace(others=[manifest_set.namespace] * 2)
+        deployer.create_resources(
+            ms=manifest_set, storage=configmap_storage, locustfile=locustfile
+        )
+        create_custom_objects.assert_called_once()
+
     @patch("zelt.kubernetes.client.config")
     @patch("zelt.kubernetes.client.CoreV1Api.create_namespace")
     @patch("zelt.kubernetes.client.CoreV1Api.create_namespaced_service")

zelt/kubernetes/client.py

@@ -14,6 +14,8 @@
     V1ContainerStatus,
     NetworkingV1beta1Api,
     NetworkingV1beta1Ingress,
+    CustomObjectsApi,
+    ApiextensionsV1beta1Api,
 )
 from kubernetes.client.rest import ApiException
 from tenacity import retry, stop_after_delay, wait_fixed, retry_if_exception_type
@@ -179,6 +181,72 @@ def delete_ingress(name: str, namespace: str) -> Optional[V1Status]:
     )
 
 
+def try_creating_custom_objects(manifests: List[Manifest]):
+    logging.info("Fetching CRDs available in the cluster...")
+    try:
+        custom_resources = (
+            ApiextensionsV1beta1Api().list_custom_resource_definition().items
+        )
+    except ApiException as err:
+        logging.error("Failed to fetch CRDs: %s", err.reason)
+        raise
+
+    available_kinds = {r.spec.names.kind.lower() for r in custom_resources}
+
+    for m in manifests:
+        logging.info("Found a custom manifest: %s %r", m.body["kind"], m.name)
+        if m.body["kind"].lower() not in available_kinds:
+            logging.error(
+                "Unsupported custom manifest %r of kind %r is ignored. "
+                "Supported custom resource types are: %s",
+                m.name,
+                m.body["kind"],
+                available_kinds,
+            )
+            continue
+
+        # By supporting only namespaced resources we don't have to manage
+        # the cleanup - it will be handled by the deletion of the namespace.
+        matching_resources = [
+            r
+            for r in custom_resources
+            if r.spec.names.kind.lower() == m.body["kind"].lower()
+            and r.spec.scope.lower() == "namespaced"
+        ]
+        if not matching_resources:
+            logging.error(
+                "Failed to match %r to a namespaced custom resource "
+                "definition. Non-namespaced resources are not supported!",
+                m.body["kind"],
+            )
+            continue
+
+        _create_custom_object_with_plural(
+            custom_object=m, plural=matching_resources[0].spec.names.plural
+        )
+
+
+def _create_custom_object_with_plural(custom_object: Manifest, plural: str):
+    logging.info("Creating %s %r ", custom_object.body["kind"], custom_object.name)
+    try:
+        group, version = custom_object.body.get("apiVersion").rsplit("/", 1)
+        return CustomObjectsApi().create_namespaced_custom_object(
+            namespace=custom_object.namespace,
+            body=custom_object.body,
+            group=group,
+            version=version,
+            plural=plural,
+        )
+    except ApiException as err:
+        logging.error(
+            "Failed to create %s %r: %s",
+            custom_object.body["kind"],
+            custom_object.name,
+            err.reason,
+        )
+        raise
+
+
 @retry(
     stop=stop_after_delay(KUBE_API_DELETE_TIMEOUT),
     wait=wait_fixed(KUBE_API_WAIT),

zelt/kubernetes/deployer.py

@@ -24,6 +24,10 @@ def create_resources(
 
         kube.create_service(ms.service)
         kube.create_ingress(ms.ingress)
+
+        if ms.others:
+            kube.try_creating_custom_objects(ms.others)
+
     except (kube.ApiException, RetryError) as err:
         logging.error("Kubernetes operation failed: %s", err.reason)