From 87685d7c3adb83113c97f3bfcae50db1368a7bfa Mon Sep 17 00:00:00 2001 From: Nelynehemia Date: Tue, 14 Mar 2023 13:04:08 +0200 Subject: [PATCH 1/7] set_directory_path test --- cloudify_kubernetes/tests/test_utils.py | 62 +++++++++++++++++++++++++ cloudify_kubernetes/utils.py | 35 ++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/cloudify_kubernetes/tests/test_utils.py b/cloudify_kubernetes/tests/test_utils.py index 0bc2f95..4b95c5c 100644 --- a/cloudify_kubernetes/tests/test_utils.py +++ b/cloudify_kubernetes/tests/test_utils.py @@ -15,6 +15,7 @@ import os import json import unittest +from tempfile import mkdtemp from mock import (MagicMock, mock_open, patch) from cloudify.state import current_ctx @@ -51,9 +52,70 @@ --- """ +test_dir1 = mkdtemp(dir='/project') + class TestUtils(unittest.TestCase): + @patch('cloudify_kubernetes.utils.get_node_instance_dir', + return_value=test_dir1) + def test_set_directory_path(*_, **__): + + + some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver' \ + '/deploy/kubernetes/overlays/stable/?ref=release-1.14' + + _ctx = MockCloudifyContext( + node_id="test_id", + node_name="test_name", + deployment_id="test_name", + properties={ + 'kustomize': some_directory + }, + ) + listdir = ['cloudbuild.yaml', + '.gitignore', + 'hack', + 'tests', + 'SECURITY_CONTACTS', + 'LICENSE', + 'README.md', + 'examples', + 'docs', + '.golangci.yml', + '.github', + 'code-of-conduct.md', + 'deploy', + 'cmd', + 'THIRD-PARTY', + 'go.mod', + 'Dockerfile', + 'charts', + 'Makefile', + 'go.sum', + 'NOTICE', + 'CONTRIBUTING.md', + 'pkg', + 'OWNERS', + 'CHANGELOG.md' + ] + + resalt = utils.set_directory_path(some_directory, + target_path=test_dir1) + print(resalt) + print(test_dir1) + assert os.listdir(resalt) == listdir + + def test_get_archive_from_github_url(*_, **__): + + some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver' \ + '/deploy/kubernetes/overlays/stable/?ref=release-1.14' + + fake_git_url = 'https://github.com/kubernetes-sigs/' \ + 'aws-ebs-csi-driver/archive/refs/heads/release-1.14.zip' + resalt = utils.get_archive_from_github_url(some_directory) + assert resalt == fake_git_url + def _assert_mapping(self, mapping): self.assertTrue( isinstance( diff --git a/cloudify_kubernetes/utils.py b/cloudify_kubernetes/utils.py index dcf82bc..7289f3e 100644 --- a/cloudify_kubernetes/utils.py +++ b/cloudify_kubernetes/utils.py @@ -16,6 +16,7 @@ import sys import json from deepdiff import DeepDiff +from urllib.parse import urlparse from tempfile import NamedTemporaryFile from collections import ( Mapping, @@ -36,6 +37,8 @@ RELATIONSHIP_INSTANCE = 'relationship-instance' from cloudify_kubernetes_sdk.state import Resource +from cloudify_common_sdk.utils import get_node_instance_dir +from cloudify_common_sdk.resource_downloader import get_shared_resource from cloudify_azure_sdk.client import AKSConnection from ._compat import text_type @@ -69,6 +72,7 @@ 'cloudify.nodes.azure.compute.ManagedCluster'] CLUSTER_REL = 'cloudify.relationships.kubernetes.connected_to_shared_cluster' DEFINITION_ADDITIONS = 'definitions_additions' +ARCHIVE_PATH = 'https://{}/archive/refs/heads/{}.zip' def merge_definitions(old, new): @@ -778,3 +782,34 @@ def update_with_additions(resource_definition, additions): def check_drift(previous, current): return DeepDiff(Resource(previous).state, Resource(current).state) + + +def get_archive_from_github_url(path): + parsed = urlparse(path) + if parsed.path.startswith('github'): + folder_path = '/'.join(parsed.path.split('/')[0:3]) + branch = parsed.query.split('=')[-1] + download_url = ARCHIVE_PATH.format(folder_path, branch) + return download_url + else: + raise NonRecoverableError('Unsupported argument: {}'.format(path)) + + +def set_directory_path(directory_path=None, target_path=None): + if not directory_path: + directory_path = ctx.node.properties['kustomize'] + + if not target_path: + target_path = get_node_instance_dir() + if 'github' in directory_path .split('/')[0]: + download_url = get_archive_from_github_url(directory_path) + get_shared_resource(download_url, target_path) + print(os.listdir(target_path)) + + elif os.path.isabs(directory_path): + ctx.download_resource(directory_path, target_path=target_path) + else: + raise NonRecoverableError('Unsupported argument: {}' + .format(directory_path)) + return target_path + From fd9aac7c3b52434431f673d1555f7898a99c2e09 Mon Sep 17 00:00:00 2001 From: Nelynehemia Date: Tue, 14 Mar 2023 14:03:00 +0200 Subject: [PATCH 2/7] test_set_directory_path --- cloudify_kubernetes/tests/test_utils.py | 56 ++++++------------------- cloudify_kubernetes/utils.py | 12 +++--- 2 files changed, 20 insertions(+), 48 deletions(-) diff --git a/cloudify_kubernetes/tests/test_utils.py b/cloudify_kubernetes/tests/test_utils.py index 4b95c5c..7aabe72 100644 --- a/cloudify_kubernetes/tests/test_utils.py +++ b/cloudify_kubernetes/tests/test_utils.py @@ -14,6 +14,7 @@ import os import json +import shutil import unittest from tempfile import mkdtemp from mock import (MagicMock, mock_open, patch) @@ -59,60 +60,29 @@ class TestUtils(unittest.TestCase): @patch('cloudify_kubernetes.utils.get_node_instance_dir', return_value=test_dir1) + @patch('cloudify_common_sdk.utils.get_deployment_dir') def test_set_directory_path(*_, **__): - - - some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver' \ - '/deploy/kubernetes/overlays/stable/?ref=release-1.14' - - _ctx = MockCloudifyContext( - node_id="test_id", - node_name="test_name", - deployment_id="test_name", - properties={ - 'kustomize': some_directory - }, - ) - listdir = ['cloudbuild.yaml', - '.gitignore', - 'hack', - 'tests', - 'SECURITY_CONTACTS', - 'LICENSE', - 'README.md', - 'examples', - 'docs', - '.golangci.yml', - '.github', - 'code-of-conduct.md', - 'deploy', - 'cmd', - 'THIRD-PARTY', - 'go.mod', - 'Dockerfile', - 'charts', - 'Makefile', - 'go.sum', - 'NOTICE', - 'CONTRIBUTING.md', - 'pkg', - 'OWNERS', - 'CHANGELOG.md' - ] + some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver/' \ + 'deploy/kubernetes/overlays/stable/?ref=release-1.14' + listdir = ['cloudbuild.yaml', 'hack', 'tests', 'SECURITY_CONTACTS', + 'LICENSE', 'README.md', 'examples', 'docs', + 'code-of-conduct.md', 'deploy', 'cmd', 'THIRD-PARTY', + 'go.mod', 'Dockerfile', 'charts', 'Makefile', 'go.sum', + 'NOTICE', 'CONTRIBUTING.md', 'pkg', 'OWNERS', + 'CHANGELOG.md'] resalt = utils.set_directory_path(some_directory, target_path=test_dir1) - print(resalt) - print(test_dir1) + print(os.listdir(resalt)) assert os.listdir(resalt) == listdir + shutil.rmtree(test_dir1) def test_get_archive_from_github_url(*_, **__): - some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver' \ '/deploy/kubernetes/overlays/stable/?ref=release-1.14' - fake_git_url = 'https://github.com/kubernetes-sigs/' \ 'aws-ebs-csi-driver/archive/refs/heads/release-1.14.zip' + resalt = utils.get_archive_from_github_url(some_directory) assert resalt == fake_git_url diff --git a/cloudify_kubernetes/utils.py b/cloudify_kubernetes/utils.py index 7289f3e..bd03c88 100644 --- a/cloudify_kubernetes/utils.py +++ b/cloudify_kubernetes/utils.py @@ -37,7 +37,10 @@ RELATIONSHIP_INSTANCE = 'relationship-instance' from cloudify_kubernetes_sdk.state import Resource -from cloudify_common_sdk.utils import get_node_instance_dir +from cloudify_common_sdk.utils import (get_node_instance_dir, + copy_directory, + remove_directory + ) from cloudify_common_sdk.resource_downloader import get_shared_resource from cloudify_azure_sdk.client import AKSConnection @@ -803,13 +806,12 @@ def set_directory_path(directory_path=None, target_path=None): target_path = get_node_instance_dir() if 'github' in directory_path .split('/')[0]: download_url = get_archive_from_github_url(directory_path) - get_shared_resource(download_url, target_path) - print(os.listdir(target_path)) - + tmp_file = get_shared_resource(download_url) + copy_directory(tmp_file, target_path) + remove_directory(tmp_file) elif os.path.isabs(directory_path): ctx.download_resource(directory_path, target_path=target_path) else: raise NonRecoverableError('Unsupported argument: {}' .format(directory_path)) return target_path - From ebf258af1f5c9d4539ea424249bda837efaeb41d Mon Sep 17 00:00:00 2001 From: Nelynehemia Date: Tue, 14 Mar 2023 14:05:14 +0200 Subject: [PATCH 3/7] update plugins yaml with cloudify.nodes.kubernetes.resources.Kustomize --- plugin.yaml | 24 ++++++++++++++++++++++++ plugin_1_4.yaml | 24 ++++++++++++++++++++++++ v2_plugin.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/plugin.yaml b/plugin.yaml index 105b9d8..01bfe4d 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -689,6 +689,30 @@ node_types: cloudify.kubernetes.resources.CustomBlueprintDefinedResource: derived_from: cloudify.nodes.kubernetes.resources.CustomBlueprintDefinedResource + cloudify.nodes.kubernetes.resources.Kustomize: + derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus + properties: + kustomize: + type: string + description: > + A path to YAML file containing the resource definition. + allow_node_redefinition: + type: boolean + description: > + Permit changing the name and kind and number of resources defined in file. + default: false + interfaces: + cloudify.interfaces.validation: + check_status: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_status + check_drift: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_drift + cloudify.interfaces.lifecycle: + create: + implementation: kubernetes.cloudify_kubernetes.tasks.create_kustomize + delete: + implementation: kubernetes.cloudify_kubernetes.tasks.delete_kustomize + cloudify.nodes.kubernetes.resources.FileDefinedResource: derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus properties: diff --git a/plugin_1_4.yaml b/plugin_1_4.yaml index 7200279..b9ae0cd 100644 --- a/plugin_1_4.yaml +++ b/plugin_1_4.yaml @@ -689,6 +689,30 @@ node_types: cloudify.kubernetes.resources.CustomBlueprintDefinedResource: derived_from: cloudify.nodes.kubernetes.resources.CustomBlueprintDefinedResource + cloudify.nodes.kubernetes.resources.Kustomize: + derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus + properties: + kustomize: + type: string + description: > + A path to YAML file containing the resource definition. + allow_node_redefinition: + type: boolean + description: > + Permit changing the name and kind and number of resources defined in file. + default: false + interfaces: + cloudify.interfaces.validation: + check_status: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_status + check_drift: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_drift + cloudify.interfaces.lifecycle: + create: + implementation: kubernetes.cloudify_kubernetes.tasks.create_kustomize + delete: + implementation: kubernetes.cloudify_kubernetes.tasks.delete_kustomize + cloudify.nodes.kubernetes.resources.FileDefinedResource: derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus properties: diff --git a/v2_plugin.yaml b/v2_plugin.yaml index 3cab588..10da335 100644 --- a/v2_plugin.yaml +++ b/v2_plugin.yaml @@ -689,6 +689,30 @@ node_types: cloudify.kubernetes.resources.CustomBlueprintDefinedResource: derived_from: cloudify.nodes.kubernetes.resources.CustomBlueprintDefinedResource + cloudify.nodes.kubernetes.resources.Kustomize: + derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus + properties: + kustomize: + type: string + description: > + A path to YAML file containing the resource definition. + allow_node_redefinition: + type: boolean + description: > + Permit changing the name and kind and number of resources defined in file. + default: false + interfaces: + cloudify.interfaces.validation: + check_status: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_status + check_drift: + implementation: kubernetes.cloudify_kubernetes.tasks.kustomize_check_drift + cloudify.interfaces.lifecycle: + create: + implementation: kubernetes.cloudify_kubernetes.tasks.create_kustomize + delete: + implementation: kubernetes.cloudify_kubernetes.tasks.delete_kustomize + cloudify.nodes.kubernetes.resources.FileDefinedResource: derived_from: cloudify.kubernetes.resources.ResourceWithValidateStatus properties: From 02fa154c4f34497758fd3bebee87f861f1037826 Mon Sep 17 00:00:00 2001 From: Nelynehemia Date: Tue, 14 Mar 2023 14:08:31 +0200 Subject: [PATCH 4/7] fix unittest --- cloudify_kubernetes/tests/test_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudify_kubernetes/tests/test_utils.py b/cloudify_kubernetes/tests/test_utils.py index 7aabe72..3d7a0c5 100644 --- a/cloudify_kubernetes/tests/test_utils.py +++ b/cloudify_kubernetes/tests/test_utils.py @@ -53,7 +53,7 @@ --- """ -test_dir1 = mkdtemp(dir='/project') +test_dir1 = mkdtemp() class TestUtils(unittest.TestCase): @@ -73,7 +73,6 @@ def test_set_directory_path(*_, **__): resalt = utils.set_directory_path(some_directory, target_path=test_dir1) - print(os.listdir(resalt)) assert os.listdir(resalt) == listdir shutil.rmtree(test_dir1) From 83c3307bfc5756d6587f812938dd65b30144b3e9 Mon Sep 17 00:00:00 2001 From: EarthmanT Date: Tue, 14 Mar 2023 09:43:21 -0400 Subject: [PATCH 5/7] suggestions --- __init__.py | 1 + cloudify_kubernetes/tests/test_utils.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..ff867b5 --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +# Created by importer \ No newline at end of file diff --git a/cloudify_kubernetes/tests/test_utils.py b/cloudify_kubernetes/tests/test_utils.py index 3d7a0c5..064c31f 100644 --- a/cloudify_kubernetes/tests/test_utils.py +++ b/cloudify_kubernetes/tests/test_utils.py @@ -53,15 +53,16 @@ --- """ -test_dir1 = mkdtemp() - class TestUtils(unittest.TestCase): - @patch('cloudify_kubernetes.utils.get_node_instance_dir', - return_value=test_dir1) + @patch('cloudify_kubernetes.utils.get_node_instance_dir') @patch('cloudify_common_sdk.utils.get_deployment_dir') - def test_set_directory_path(*_, **__): + def test_set_directory_path(self, *args, **__): + get_node_instance_dir = args[1] + test_dir1 = mkdtemp() + get_node_instance_dir.return_value = test_dir1 + some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver/' \ 'deploy/kubernetes/overlays/stable/?ref=release-1.14' listdir = ['cloudbuild.yaml', 'hack', 'tests', 'SECURITY_CONTACTS', @@ -71,9 +72,9 @@ def test_set_directory_path(*_, **__): 'NOTICE', 'CONTRIBUTING.md', 'pkg', 'OWNERS', 'CHANGELOG.md'] - resalt = utils.set_directory_path(some_directory, + result = utils.set_directory_path(some_directory, target_path=test_dir1) - assert os.listdir(resalt) == listdir + self.assertEqual(sorted(os.listdir(result)), sorted(listdir)) shutil.rmtree(test_dir1) def test_get_archive_from_github_url(*_, **__): From 0f9ec0ed0fd232388a4589c6edf98f2005693f80 Mon Sep 17 00:00:00 2001 From: EarthmanT Date: Tue, 14 Mar 2023 09:48:39 -0400 Subject: [PATCH 6/7] Delete __init__.py --- __init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index ff867b5..0000000 --- a/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Created by importer \ No newline at end of file From c1c2c887fbb127962262b79fe773c1ff4392c0ef Mon Sep 17 00:00:00 2001 From: Nelynehemia Date: Tue, 14 Mar 2023 16:11:48 +0200 Subject: [PATCH 7/7] fix CR --- cloudify_kubernetes/tests/test_utils.py | 5 ++--- cloudify_kubernetes/utils.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cloudify_kubernetes/tests/test_utils.py b/cloudify_kubernetes/tests/test_utils.py index 064c31f..8c06077 100644 --- a/cloudify_kubernetes/tests/test_utils.py +++ b/cloudify_kubernetes/tests/test_utils.py @@ -66,11 +66,10 @@ def test_set_directory_path(self, *args, **__): some_directory = 'github.com/kubernetes-sigs/aws-ebs-csi-driver/' \ 'deploy/kubernetes/overlays/stable/?ref=release-1.14' listdir = ['cloudbuild.yaml', 'hack', 'tests', 'SECURITY_CONTACTS', - 'LICENSE', 'README.md', 'examples', 'docs', + 'LICENSE', 'README.md', 'examples', 'docs', 'CHANGELOG.md', 'code-of-conduct.md', 'deploy', 'cmd', 'THIRD-PARTY', 'go.mod', 'Dockerfile', 'charts', 'Makefile', 'go.sum', - 'NOTICE', 'CONTRIBUTING.md', 'pkg', 'OWNERS', - 'CHANGELOG.md'] + 'NOTICE', 'CONTRIBUTING.md', 'pkg', 'OWNERS'] result = utils.set_directory_path(some_directory, target_path=test_dir1) diff --git a/cloudify_kubernetes/utils.py b/cloudify_kubernetes/utils.py index bd03c88..283c981 100644 --- a/cloudify_kubernetes/utils.py +++ b/cloudify_kubernetes/utils.py @@ -795,23 +795,26 @@ def get_archive_from_github_url(path): download_url = ARCHIVE_PATH.format(folder_path, branch) return download_url else: - raise NonRecoverableError('Unsupported argument: {}'.format(path)) + raise NonRecoverableError( + 'Unsupported argument: {}.' + 'for Kustomize directory. ' + 'At the moment, only Github refs are supported'.format(path)) -def set_directory_path(directory_path=None, target_path=None): - if not directory_path: - directory_path = ctx.node.properties['kustomize'] +def set_directory_path(dir_path=None, target_path=None): + if not dir_path: + dir_path = ctx.node.properties['kustomize'] if not target_path: target_path = get_node_instance_dir() - if 'github' in directory_path .split('/')[0]: - download_url = get_archive_from_github_url(directory_path) + if 'github' in dir_path.split('/')[0]: + download_url = get_archive_from_github_url(dir_path) tmp_file = get_shared_resource(download_url) copy_directory(tmp_file, target_path) remove_directory(tmp_file) - elif os.path.isabs(directory_path): - ctx.download_resource(directory_path, target_path=target_path) else: - raise NonRecoverableError('Unsupported argument: {}' - .format(directory_path)) + raise NonRecoverableError( + 'Unsupported argument: {}. ' + 'for Kustomize directory. ' + 'At the moment, only Github refs are supported'.format(dir_path)) return target_path