diff --git a/Makefile b/Makefile index 17fb0fa65ec..6cc32b733bc 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SELF_MAKE := $(lastword $(MAKEFILE_LIST)) -PKG_SET = tools/c7n_tencent tools/c7n_huawei tools/c7n_aliyun tools/c7n_gcp tools/c7n_azure tools/c7n_kube tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext +PKG_SET = tools/c7n_tencent tools/c7n_huawei tools/c7n_aliyun tools/c7n_gcp tools/c7n_azure tools/c7n_kube tools/c7n_openstack tools/c7n_vsphere tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext install: python3 -m venv . diff --git a/build.sh b/build.sh index 37a4f94521c..548f92f6cc6 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ echo "构建custodian镜像 ..." -docker build -t registry.cn-qingdao.aliyuncs.com/x-lab/riskscanner/custodian:master . -docker push registry.cn-qingdao.aliyuncs.com/x-lab/riskscanner/custodian:master +docker build -t registry.cn-qingdao.aliyuncs.com/x-lab/custodian:1.4 . +docker push registry.cn-qingdao.aliyuncs.com/x-lab/custodian:1.4 diff --git a/c7n/commands.py b/c7n/commands.py index 77c82b9364d..9f0c190c9d3 100644 --- a/c7n/commands.py +++ b/c7n/commands.py @@ -553,4 +553,14 @@ def version_cmd(options): packages.append('c7n_azure') if 'k8s' in found: packages.append('c7n_kube') + if 'aliyun' in found: + packages.append('c7n_aliyun') + if 'tencent' in found: + packages.append('c7n_tencent') + if 'huawei' in found: + packages.append('c7n_huawei') + if 'openstack' in found: + packages.append('c7n_openstack') + if 'vsphere' in found: + packages.append('c7n_vsphere') print(generate_requirements(packages)) diff --git a/c7n/resources/__init__.py b/c7n/resources/__init__.py index 2338e942354..e67abc55360 100644 --- a/c7n/resources/__init__.py +++ b/c7n/resources/__init__.py @@ -49,7 +49,7 @@ def should_load_provider(name, provider_types): return False -PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'aliyun', 'huawei', 'tencent') +PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'aliyun', 'huawei', 'tencent', 'openstack', 'vsphere') def load_available(resources=True): @@ -73,7 +73,6 @@ def load_available(resources=True): def load_providers(provider_types): global LOADED - # Even though we're lazy loading resources we still need to import # those that are making available generic filters/actions if should_load_provider('aws', provider_types): @@ -108,4 +107,11 @@ def load_providers(provider_types): from c7n_tencent.entry import initialize_tencent initialize_tencent() + if should_load_provider('openstack', provider_types): + from c7n_openstack.entry import initialize_openstack + initialize_openstack() + + if should_load_provider('vsphere', provider_types): + from c7n_vsphere.entry import initialize_vsphere + initialize_vsphere() LOADED.update(provider_types) diff --git a/docker/cli b/docker/cli index 0b5ccc4558f..871652c5820 100644 --- a/docker/cli +++ b/docker/cli @@ -1,4 +1,4 @@ -FROM registry.fit2cloud.com/fit2cloud2/fabric8-java-alpine-openjdk8-jre:py37 as build-env +FROM registry.cn-qingdao.aliyuncs.com/x-lab/fabric8-java-alpine-openjdk8-jre:py37 as build-env USER root @@ -8,7 +8,7 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories # pre-requisite distro deps, and build env setup RUN apk add --update \ curl \ - && apk add gcc g++ make libffi-dev openssl-dev libtool --no-cache \ + && apk add gcc g++ make git libffi-dev openssl-dev libxml2-dev libxslt-dev musl-dev cargo libtool --no-cache \ && python3 -m venv /usr/local \ && curl -sSL https://nginx-qa.fit2cloud.com/tools/get-poetry.py | python3 @@ -26,17 +26,23 @@ ADD tools/c7n_gcp /src/tools/c7n_gcp RUN rm -R tools/c7n_gcp/tests ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure -ADD tools/c7n_kube /src/tools/c7n_kube -RUN rm -R tools/c7n_kube/tests +#ADD tools/c7n_kube /src/tools/c7n_kube +#RUN rm -R tools/c7n_kube/tests ADD tools/c7n_aliyun /src/tools/c7n_aliyun RUN rm -R tools/c7n_aliyun/test ADD tools/c7n_huawei /src/tools/c7n_huawei RUN rm -R tools/c7n_huawei/test ADD tools/c7n_tencent /src/tools/c7n_tencent RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube aliyun huawei tencent" +ARG providers="azure gcp aliyun huawei tencent openstack vsphere" +RUN . /usr/local/bin/activate && \ + pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry lock && \ @@ -44,13 +50,13 @@ RUN . /usr/local/bin/activate && \ RUN mkdir /output -FROM registry.fit2cloud.com/fit2cloud2/fabric8-java-alpine-openjdk8-jre:py37 +FROM registry.cn-qingdao.aliyuncs.com/x-lab/fabric8-java-alpine-openjdk8-jre:py37 COPY --from=build-env /src /src COPY --from=build-env /usr/local /usr/local COPY --from=build-env /output /output -RUN rm -Rf /var/cache/apk +RUN rm -Rf /var/cache/apk/* USER root WORKDIR /home/custodian diff --git a/docker/cli-distroless b/docker/cli-distroless index c3df08d8c48..4b462cba277 100644 --- a/docker/cli-distroless +++ b/docker/cli-distroless @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/mailer b/docker/mailer index 20a221a1806..35d2b5ee695 100644 --- a/docker/mailer +++ b/docker/mailer @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/mailer-distroless b/docker/mailer-distroless index b8d9881e93b..09e45496e72 100644 --- a/docker/mailer-distroless +++ b/docker/mailer-distroless @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/org b/docker/org index 5651be72669..972bb653647 100644 --- a/docker/org +++ b/docker/org @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/org-distroless b/docker/org-distroless index f4758489dfc..d99e7d159e2 100644 --- a/docker/org-distroless +++ b/docker/org-distroless @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/policystream b/docker/policystream index c91284b1687..b2bfdc4dc74 100644 --- a/docker/policystream +++ b/docker/policystream @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/docker/policystream-distroless b/docker/policystream-distroless index 3983b8d8ccd..8edd4b47b4f 100644 --- a/docker/policystream-distroless +++ b/docker/policystream-distroless @@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests +ADD tools/c7n_aliyun /src/tools/c7n_aliyun +RUN rm -R tools/c7n_aliyun/test +ADD tools/c7n_huawei /src/tools/c7n_huawei +RUN rm -R tools/c7n_huawei/test +ADD tools/c7n_tencent /src/tools/c7n_tencent +RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_openstack /src/tools/c7n_openstack +RUN rm -R tools/c7n_openstack/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \ for pkg in $providers; do cd tools/c7n_$pkg && \ $HOME/.poetry/bin/poetry install && cd ../../; done diff --git a/requirements-dev.txt b/requirements-dev.txt index 4ca0e19a65f..e5d3f7f73a8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,6 +8,10 @@ -r tools/c7n_mailer/requirements.txt -r tools/c7n_org/requirements.txt -r tools/c7n_aliyun/requirements.txt +-r tools/c7n_huawei/requirements.txt +-r tools/c7n_tencent/requirements.txt +-r tools/c7n_openstack/requirements.txt +-r tools/c7n_vsphere/requirements.txt # Setup source directories as editable/development distributions -e . @@ -23,6 +27,14 @@ -e tools/c7n_org # Local package required for c7n_aliyun tests -e tools/c7n_aliyun +# Local package required for c7n_huawei tests +-e tools/c7n_huawei +# Local package required for c7n_tencent tests +-e tools/c7n_tencent +# Local package required for c7n_vsphere tests +-e tools/c7n_openstack +# Local package required for c7n_vsphere tests +-e tools/c7n_vsphere # we don't export dev requirements of subpackages (due to editable/distribution above) # so explicitly list them here.. diff --git a/tests/test_docker.py b/tests/test_docker.py index 64847490cbb..33547300db2 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -162,7 +162,7 @@ def test_image_metadata(image_name): def test_cli_providers_available(): providers = os.environ.get("CUSTODIAN_PROVIDERS", None) if providers is None: - providers = {"aws", "azure", "gcp", "k8s"} + providers = {"aws", "azure", "gcp", "k8s", "aliyun", "huawei", "tencent", "openstack", "vsphere"} elif providers == "": providers = {"aws"} else: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index cae6b2213f7..a1e5c2f2ed0 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -19,7 +19,8 @@ @pytest.mark.parametrize("package", [ "c7n", "c7n_azure", "c7n_gcp", "c7n_kube", "c7n_org", "c7n_mailer", "policystream", "c7n_trailcreator", - "c7n_logexporter", "c7n_sphinxext"]) + "c7n_logexporter", "c7n_sphinxext", "c7n_aliyun", + "c7n_huawei", "c7n_tencent", "c7n_vsphere", "c7n_vsphere"]) def test_package_metadata(package): try: m = __import__(package) diff --git a/tools/c7n_aliyun/.gitignore b/tools/c7n_aliyun/.gitignore index 1e7bb232c6a..36ec3a3b659 100644 --- a/tools/c7n_aliyun/.gitignore +++ b/tools/c7n_aliyun/.gitignore @@ -2,3 +2,4 @@ *py~ __pycache__ *.egg-info +aliyun.py diff --git a/tools/c7n_aliyun/c7n_aliyun/client.py b/tools/c7n_aliyun/c7n_aliyun/client.py index d743cdc8b47..d39bfc77dca 100644 --- a/tools/c7n_aliyun/c7n_aliyun/client.py +++ b/tools/c7n_aliyun/c7n_aliyun/client.py @@ -61,26 +61,26 @@ def get_oss_region(self, regionId): return region REGION_ENDPOINT = { - 'cn-hangzhou': 'oss-cn-hangzhou.aliyuncs.com', - 'cn-shanghai': 'oss-cn-shanghai.aliyuncs.com', - 'cn-qingdao': 'oss-cn-qingdao.aliyuncs.com', - 'cn-beijing': 'oss-cn-beijing.aliyuncs.com', - 'cn-zhangjiakou': 'oss-cn-zhangjiakou.aliyuncs.com', - 'cn-huhehaote': 'oss-cn-huhehaote.aliyuncs.com', - 'cn-shenzhen': 'oss-cn-shenzhen.aliyuncs.com', - 'cn-hongkong': 'oss-cn-hongkong.aliyuncs.com', - 'us-west-1': 'oss-us-west-1.aliyuncs.com', - 'us-east-1': 'oss-us-east-1.aliyuncs.com', - 'ap-southeast-1': 'oss-ap-southeast-1.aliyuncs.com', - 'ap-southeast-2': 'oss-ap-southeast-2.aliyuncs.com', - 'ap-southeast-3': 'oss-ap-southeast-3.aliyuncs.com', - 'ap-southeast-5': 'oss-ap-southeast-5.aliyuncs.com', - 'ap-northeast-1': 'oss-ap-northeast-1.aliyuncs.com', - 'ap-south-1': 'oss-ap-south-1.aliyuncs.com', - 'eu-central-1': 'oss-eu-central-1.aliyuncs.com', - 'eu-west-1': 'oss-eu-west-1.aliyuncs.com', - 'me-east-1': 'oss-me-east-1.aliyuncs.com', - 'cn-wulanchabu': 'oss-cn-wulanchabu.aliyuncs.com', - 'cn-heyuan': 'oss-cn-heyuan.aliyuncs.com', - 'cn-chengdu': 'oss-cn-chengdu.aliyuncs.com' - } \ No newline at end of file + 'cn-hangzhou': 'oss-cn-hangzhou.aliyuncs.com', + 'cn-shanghai': 'oss-cn-shanghai.aliyuncs.com', + 'cn-qingdao': 'oss-cn-qingdao.aliyuncs.com', + 'cn-beijing': 'oss-cn-beijing.aliyuncs.com', + 'cn-zhangjiakou': 'oss-cn-zhangjiakou.aliyuncs.com', + 'cn-huhehaote': 'oss-cn-huhehaote.aliyuncs.com', + 'cn-shenzhen': 'oss-cn-shenzhen.aliyuncs.com', + 'cn-hongkong': 'oss-cn-hongkong.aliyuncs.com', + 'us-west-1': 'oss-us-west-1.aliyuncs.com', + 'us-east-1': 'oss-us-east-1.aliyuncs.com', + 'ap-southeast-1': 'oss-ap-southeast-1.aliyuncs.com', + 'ap-southeast-2': 'oss-ap-southeast-2.aliyuncs.com', + 'ap-southeast-3': 'oss-ap-southeast-3.aliyuncs.com', + 'ap-southeast-5': 'oss-ap-southeast-5.aliyuncs.com', + 'ap-northeast-1': 'oss-ap-northeast-1.aliyuncs.com', + 'ap-south-1': 'oss-ap-south-1.aliyuncs.com', + 'eu-central-1': 'oss-eu-central-1.aliyuncs.com', + 'eu-west-1': 'oss-eu-west-1.aliyuncs.com', + 'me-east-1': 'oss-me-east-1.aliyuncs.com', + 'cn-wulanchabu': 'oss-cn-wulanchabu.aliyuncs.com', + 'cn-heyuan': 'oss-cn-heyuan.aliyuncs.com', + 'cn-chengdu': 'oss-cn-chengdu.aliyuncs.com' +} \ No newline at end of file diff --git a/tools/c7n_aliyun/c7n_aliyun/filters/filter.py b/tools/c7n_aliyun/c7n_aliyun/filters/filter.py index 1456589336c..3334762cae5 100644 --- a/tools/c7n_aliyun/c7n_aliyun/filters/filter.py +++ b/tools/c7n_aliyun/c7n_aliyun/filters/filter.py @@ -1,6 +1,5 @@ import datetime import json -import logging from concurrent.futures import as_completed from datetime import timedelta @@ -308,7 +307,7 @@ def process(self, resources, event=None): fattrs = list(sorted(self.perm_attrs.intersection(self.data.keys()))) self.ports = 'Ports' in self.data and self.data['Ports'] or () self.only_ports = ( - 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) + 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) for f in fattrs: fv = self.data.get(f) if isinstance(fv, dict): @@ -462,13 +461,15 @@ class MetricsFilter(Filter): """Supports metrics filters on resources. .. code-block:: yaml - - name: ecs-underutilized - resource: ecs + policies: + - name: aliyun-ecs-underutilized + resource: aliyun.ecs filters: - type: metrics name: CPUUtilization - days: 4 + days: 7 period: 86400 + statistics: Average value: 30 op: less-than @@ -483,8 +484,9 @@ class MetricsFilter(Filter): .. code-block:: yaml - - name: elb-low-request-count - resource: elb + policies: + - name: aliyun-elb-low-request-count + resource: aliyun.elb filters: - type: metrics name: RequestCount @@ -575,7 +577,12 @@ def process(self, resources, event=None): return matched def get_dimensions(self, resource): - return [{self.model.dimension: resource[self.model.dimension]}] + start = self.model.dimension[:1] + end = self.model.dimension[1:] + key = start.lower() + end + if key!='instanceId': + key = 'instanceId' + return [{key: resource[self.model.dimension]}] def get_user_dimensions(self): dims = [] @@ -625,7 +632,7 @@ def process_resource_set(self, resource_set): collected_metrics[key].append({'timestamp': self.start, self.statistics: self.data['missing-value'], 'c7n_aliyun:detail': 'Fill value for missing data'}) else: collected_metrics[key].append({'startTime': self.start, 'endTime': self.end, self.statistics: 0, - 'detail': 'The read and write monitoring value within the specified time range is 0 (or no data)'}) + 'detail': 'The read and write monitoring value within the specified time range is 0 (or no data)'}) if self.model.service == "disk": for data in collected_metrics[key]: if self.op(data[data_usage], self.value): diff --git a/tools/c7n_aliyun/c7n_aliyun/query.py b/tools/c7n_aliyun/c7n_aliyun/query.py index 8e68cd3a9ad..5e38cd25c8e 100644 --- a/tools/c7n_aliyun/c7n_aliyun/query.py +++ b/tools/c7n_aliyun/c7n_aliyun/query.py @@ -39,27 +39,66 @@ def filter(self, resource_manager, **params): if extra_args: params.update(extra_args) + pageNumber = 1 + pageSize = 100 + res = [] + buckets = [] if m.service == 'oss': - request = resource_manager.get_request() - result = client.list_buckets() - buckets = [] - for b in result.buckets: - if request in b.__dict__['location']: - b.__dict__['F2CId'] = b.__dict__[m.id] - buckets.append(b.__dict__) + marker_param = "" + while 1 <= pageNumber: + request = resource_manager.get_request() + result = client.list_buckets(marker=marker_param) + for b in result.buckets: + if request in b.__dict__['location']: + b.__dict__['F2CId'] = b.__dict__[m.id] + buckets.append(b.__dict__) + if len(result.buckets) == pageSize: + marker_param = result.next_marker + else: + return buckets return buckets - else: - request = resource_manager.get_request() - if request: + elif m.service == 'ram': + marker_param = None + while 1 <= pageNumber: + request = resource_manager.get_request() + request.set_accept_format('json') + if marker_param is not None: + request.set_Marker(marker_param) result = client.do_action_with_exception(request) false = "false" true = "true" - res = jmespath.search(path, eval(result)) - for data in res: + response = jmespath.search(path, eval(result)) + for data in response: data['F2CId'] = data[m.id] - return res - else: - return None + res = res + response + if len(response) == pageSize: + marker_param = eval(result).get('Marker', None) + else: + return res + return res + else: + + while 1 <= pageNumber: + request = resource_manager.get_request() + request.set_accept_format('json') + request.set_PageSize(pageSize) + request.set_PageNumber(pageNumber) + if request: + result = client.do_action_with_exception(request) + false = "false" + true = "true" + response = jmespath.search(path, eval(result)) + for data in response: + data['F2CId'] = data[m.id] + else: + response = [] + res = res + response + if len(response) == pageSize: + pageNumber += 1 + else: + return res + return res + def _invoke_client_enum(self, client, request, params, path): result = client.do_action_with_exception(request) diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/disk.py b/tools/c7n_aliyun/c7n_aliyun/resources/disk.py index 9a81d2eba42..d5f5f1e04dc 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/disk.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/disk.py @@ -58,7 +58,8 @@ class AliyunDiskFilter(AliyunDiskFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if i['Status'] != type: + encrypted = self.data.get('type', '') + if i.get('Encrypted', '').lower() != str(encrypted).lower(): return False return i @@ -83,8 +84,7 @@ class AliyunDiskFilter(AliyunDiskFilter): schema = type_schema('Available') def get_request(self, i): - encrypted = self.data['value'] - if i['Encrypted'].lower() != str(encrypted).lower(): + if i.get('Status', '') != 'Available': return False return i diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/ecs.py b/tools/c7n_aliyun/c7n_aliyun/resources/ecs.py index 53311a1e534..2407cb006a5 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/ecs.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/ecs.py @@ -106,6 +106,7 @@ def get_request(self, r): request.set_MetricName(self.metric) return request + @Ecs.filter_registry.register('instance-network-type') class InstanceNetworkTypeEcsFilter(AliyunEcsFilter): """Filters @@ -132,3 +133,86 @@ def get_request(self, i): if self.data['value'] == i['InstanceNetworkType']: return False return i + +@Ecs.filter_registry.register('vpc-type') +class VpcTypeEcsFilter(AliyunEcsFilter): + """Filters + :Example: + .. code-block:: yaml + + policies: + # 检测您账号下的Ecs实例指定属于哪些VPC, 属于则合规，不属于则"不合规"。 + - name: aliyun-ecs-vpc-type + resource: aliyun.ecs + filters: + - type: vpc-type + vpcIds: ["111", "222"] + """ + schema = type_schema( + 'vpc-type', + **{'vpcIds': {'type': 'array', 'items': {'type': 'string'}}}) + + def get_request(self, i): + vpcId = i['VpcAttributes']['VpcId'] + if vpcId in self.data['vpcIds']: + return False + return i + +@Ecs.filter_registry.register('stopped') +class AliyunEcsFilter(AliyunEcsFilter): + """Filters + + :Example: + + .. code-block:: yaml + + policies: + - name: aliyun-ecs + resource: aliyun.ecs + filters: + - type: stopped + """ + # 实例状态。取值范围： + # + # Pending：创建中 + # Running：运行中 + # Starting：启动中 + # Stopping：停止中 + # Stopped：已停止 + schema = type_schema('Stopped') + + def get_request(self, i): + status = i.get('Status', '') + if 'Stopped' != status: + return False + return i + +@Ecs.filter_registry.register('instance-age') +class EcsAgeFilter(AliyunAgeFilter): + """Filters instances based on their age (in days) + + :Example: + + .. code-block:: yaml + + policies: + - name: ecs-30-days-plus + resource: aliyun.ecs + filters: + - type: instance-age + op: ge + minutes: 1 + """ + + date_attribute = "LaunchTime" + ebs_key_func = operator.itemgetter('AttachTime') + + schema = type_schema( + 'instance-age', + op={'$ref': '#/definitions/filters_common/comparison_operators'}, + days={'type': 'number'}, + hours={'type': 'number'}, + minutes={'type': 'number'}) + + def get_resource_date(self, i): + return i['CreationTime'] \ No newline at end of file diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/mongodb.py b/tools/c7n_aliyun/c7n_aliyun/resources/mongodb.py index 36742dbe16b..2a87ec8704f 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/mongodb.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/mongodb.py @@ -14,12 +14,8 @@ import logging import os -import jmespath -from aliyunsdkr_kvstore.request.v20150101.DescribeInstancesRequest import DescribeInstancesRequest -from aliyunsdkdds.request.v20151201.DescribeSecurityIpsRequest import DescribeSecurityIpsRequest -from aliyunsdkdds.request.v20151201.DescribeSecurityGroupConfigurationRequest import DescribeSecurityGroupConfigurationRequest - - +from aliyunsdkdds.request.v20151201.DescribeDBInstancesRequest import DescribeDBInstancesRequest +from aliyunsdkdds.request.v20151201.DescribeShardingNetworkAddressRequest import DescribeShardingNetworkAddressRequest from c7n.utils import type_schema from c7n_aliyun.filters.filter import AliyunRdsFilter @@ -37,11 +33,11 @@ class MongoDB(QueryResourceManager): class resource_type(TypeInfo): service = 'mongodb' - enum_spec = (None, 'Instances.KVStoreInstance', None) + enum_spec = (None, 'DBInstances.DBInstance', None) id = 'InstanceId' def get_request(self): - return DescribeInstancesRequest() + return DescribeDBInstancesRequest() @MongoDB.filter_registry.register('network-type') class NetworkTypeMongoDBFilter(AliyunRdsFilter): @@ -89,23 +85,28 @@ class InternetAccessMongoDBFilter(AliyunRdsFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - request1 = DescribeSecurityIpsRequest() - request1.set_accept_format('json') - response1 = Session.client(self, service).do_action_with_exception(request1) - string1 = str(response1, encoding="utf-8").replace("false", "False") - SecurityIpGroups = jmespath.search('SecurityIpGroups.SecurityIpGroup', eval(string1)) - request2 = DescribeSecurityGroupConfigurationRequest() - request2.set_accept_format('json') - response2 = Session.client(self, service).do_action_with_exception(request2) - string2 = str(response2, encoding="utf-8").replace("false", "False") - - RdsEcsSecurityGroupRel = jmespath.search('Items.RdsEcsSecurityGroupRel', eval(string2)) - if self.data['value']: - if len(SecurityIpGroups) == 0 and len(RdsEcsSecurityGroupRel) == 0: - return False + request = DescribeShardingNetworkAddressRequest() + request.set_accept_format('json') + + request.set_DBInstanceId(i.get('DBInstanceId', '')) + response = Session.client(self, service).do_action_with_exception(request) + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") + data = eval(string) + CompatibleConnections = data.get('CompatibleConnections', {}).get('CompatibleConnection', []) + NetworkAddresses = data.get('NetworkAddresses', {}).get('NetworkAddress', []) + if self.data.get('value', '') == True: + for c in CompatibleConnections: + if c.get('NetworkType', '') == 'Public': + return i + for n in NetworkAddresses: + if n.get('NetworkType', '') == 'Public': + return i else: - return False - i['SecurityIpGroups'] = SecurityIpGroups - i['RdsEcsSecurityGroupRel'] = RdsEcsSecurityGroupRel - return i \ No newline at end of file + for c in CompatibleConnections: + if c.get('NetworkType', '') != 'Public': + return i + for n in NetworkAddresses: + if n.get('NetworkType', '') != 'Public': + return i + return None \ No newline at end of file diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/oss.py b/tools/c7n_aliyun/c7n_aliyun/resources/oss.py index e6cc3f0e74e..ba869296069 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/oss.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/oss.py @@ -179,7 +179,7 @@ class BucketRefererOssFilter(AliyunOssFilter): def get_request(self, i): try: auth = oss2.Auth(accessKeyId, accessSecret) - bucket = oss2.Bucket(auth, i['extranet_endpoint'], i['name']) + bucket = oss2.Bucket(auth, i.get('extranet_endpoint', ''), i.get('name', '')) result = bucket.get_bucket_referer() if self.data['value']: if result.referers: diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/polardb.py b/tools/c7n_aliyun/c7n_aliyun/resources/polardb.py index 5c6eaccfe94..32cb855344a 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/polardb.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/polardb.py @@ -40,6 +40,30 @@ class resource_type(TypeInfo): def get_request(self): return DescribeDBClustersRequest() +@PolarDB.filter_registry.register('vpc-type') +class VpcTypePolarDBFilter(AliyunRdsFilter): + """Filters + :Example: + .. code-block:: yaml + + policies: + # 检测您账号下的Polardb实例指定属于哪些VPC, 属于则合规，不属于则"不合规"。 + - name: aliyun-polardb-vpc-type + resource: aliyun.polardb + filters: + - type: vpc-type + vpcIds: ["111", "222"] + """ + schema = type_schema( + 'vpc-type', + **{'vpcIds': {'type': 'array', 'items': {'type': 'string'}}}) + + def get_request(self, i): + vpcId = i['VpcId'] + if vpcId in self.data['vpcIds']: + return False + return i + @PolarDB.filter_registry.register('dbcluster-network-type') class DBClusterNetworkTypePolarDBFilter(AliyunRdsFilter): """Filters @@ -88,9 +112,9 @@ class InternetAccessPolarDBFilter(AliyunRdsFilter): def get_request(self, i): request = DescribeDBClusterAccessWhitelistRequest() request.set_accept_format('json') - request.set_DBClusterId(i['DBClusterId']) + request.set_DBClusterId(i.get('DBClusterId', '')) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) DBClusterSecurityGroups = jmespath.search('DBClusterSecurityGroups.DBClusterSecurityGroup', data) DBClusterIPArray = jmespath.search('Items.DBClusterIPArray', data) diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/ram.py b/tools/c7n_aliyun/c7n_aliyun/resources/ram.py index 3d7e762b43d..2ab7a086240 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/ram.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/ram.py @@ -61,7 +61,7 @@ def get_request(self, i): request.set_accept_format('json') request.set_UserName(i['UserName']) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = jmespath.search(self.mfa_bind_required, eval(string)) if data != self.data['value']: return False diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/rds.py b/tools/c7n_aliyun/c7n_aliyun/resources/rds.py index 34c91b05b2c..b0907b3d29a 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/rds.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/rds.py @@ -44,7 +44,6 @@ class resource_type(TypeInfo): def get_request(self): request = DescribeDBInstancesRequest() - request.set_accept_format('json') return request @Rds.filter_registry.register('available-zones') @@ -79,7 +78,7 @@ def get_request(self, i): response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) if self.data['value']==True: flag = len(data['AvailableZones']) > 0 @@ -117,7 +116,7 @@ def get_request(self, i): request.set_DBInstanceId(i['DBInstanceId']) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) DBInstanceAttributes = data['Items']['DBInstanceAttribute'] for obj in DBInstanceAttributes: @@ -154,7 +153,7 @@ def get_request(self, i): request.set_DBInstanceId(i['DBInstanceId']) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) DBInstanceAttributes = data['Items']['DBInstanceAttribute'] for obj in DBInstanceAttributes: @@ -187,6 +186,8 @@ class RdsMetricsFilter(MetricsFilter): def get_request(self, rds): request = DescribeMetricListRequest() request.set_accept_format('json') + dimensions = self.get_dimensions(rds) + request.set_Dimensions(dimensions) request.set_StartTime(self.start) request.set_Period(self.period) request.set_Namespace(self.namespace) @@ -221,6 +222,30 @@ def get_request(self, i): return False return i +@Rds.filter_registry.register('vpc-type') +class VpcTypeRdsFilter(AliyunRdsFilter): + """Filters + :Example: + .. code-block:: yaml + + policies: + # 检测您账号下的Rds实例指定属于哪些VPC, 属于则合规，不属于则"不合规"。 + - name: aliyun-rds-vpc-type + resource: aliyun.rds + filters: + - type: vpc-type + vpcIds: ["111", "222"] + """ + schema = type_schema( + 'vpc-type', + **{'vpcIds': {'type': 'array', 'items': {'type': 'string'}}}) + + def get_request(self, i): + vpcId = i['VpcId'] + if vpcId in self.data['vpcIds']: + return False + return i + @Rds.filter_registry.register('instance-network-type') class InstanceNetworkTypeRdsFilter(AliyunRdsFilter): """Filters @@ -267,26 +292,27 @@ class InternetAccessRdsFilter(AliyunRdsFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - request1 = DescribeSecurityGroupConfigurationRequest() - request1.set_accept_format('json') - request1.set_DBInstanceId(i['DBInstanceId']) - response1 = Session.client(self, service).do_action_with_exception(request1) - - string1 = str(response1, encoding="utf-8").replace("false", "False") - EcsSecurityGroupRelation = jmespath.search('Items.EcsSecurityGroupRelation', eval(string1)) - - request2 = DescribeDBInstanceIPArrayListRequest() - request2.set_accept_format('json') - request2.set_DBInstanceId(i['DBInstanceId']) - response2 = Session.client(self, service).do_action_with_exception(request2) - string2 = str(response2, encoding="utf-8").replace("false", "False") - - DBInstanceIPArray = jmespath.search('Items.DBInstanceIPArray', eval(string2)) - if self.data['value']: - if len(EcsSecurityGroupRelation) == 0 and len(DBInstanceIPArray) == 0: - return False + DBInstanceNetType = i.get('DBInstanceNetType', '') + if self.data.get('value', ''): + if DBInstanceNetType == "Internet": + return i + else: + return None else: - return False - i['EcsSecurityGroupRelation'] = EcsSecurityGroupRelation - i['DBInstanceIPArray'] = DBInstanceIPArray - return i \ No newline at end of file + if DBInstanceNetType == "Intranet": + return i + else: + return None + return i + + def is_internal_ip(self, ip): + ip = self.ip_into_int(ip) + net_a = self.ip_into_int('10.255.255.255') >> 24 + net_b = self.ip_into_int('172.31.255.255') >> 20 + net_c = self.ip_into_int('192.168.255.255') >> 16 + return ip >> 24 == net_a or ip >>20 == net_b or ip >> 16 == net_c + + def ip_into_int(self, ip): + # 先把 192.168.31.46 用map分割'.'成数组，然后用reduuce+lambda转成10进制的 3232243502 + # (((((192 * 256) + 168) * 256) + 31) * 256) + 46 + return self.reduce(lambda x,y:(x<<8)+y,map(int,ip.split('.'))) \ No newline at end of file diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/redis.py b/tools/c7n_aliyun/c7n_aliyun/resources/redis.py index 811899ce0f5..17232500a9f 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/redis.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/redis.py @@ -88,23 +88,16 @@ class InternetAccessMongoDBFilter(AliyunRedisFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - request1 = DescribeSecurityIpsRequest() - request1.set_accept_format('json') - response1 = Session.client(self, service).do_action_with_exception(request1) - string1 = str(response1, encoding="utf-8").replace("false", "False") - SecurityIpGroups = jmespath.search('SecurityIpGroups.SecurityIpGroup', eval(string1)) - request2 = DescribeSecurityGroupConfigurationRequest() - request2.set_accept_format('json') - response2 = Session.client(self, service).do_action_with_exception(request2) - string2 = str(response2, encoding="utf-8").replace("false", "False") - - EcsSecurityGroupRelation = jmespath.search('Items.EcsSecurityGroupRelation', eval(string2)) - if self.data['value']: - if len(SecurityIpGroups) == 0 and len(EcsSecurityGroupRelation) == 0: - return False + DBInstanceNetType = i.get('DBInstanceNetType', '') + if self.data.get('value', ''): + if DBInstanceNetType == "Internet": + return i + else: + return None else: - return False - i['SecurityIpGroups'] = SecurityIpGroups - i['EcsSecurityGroupRelation'] = EcsSecurityGroupRelation + if DBInstanceNetType == "Intranet": + return i + else: + return None return i \ No newline at end of file diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/slb.py b/tools/c7n_aliyun/c7n_aliyun/resources/slb.py index 2057ad0de8d..8638b835ca1 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/slb.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/slb.py @@ -21,6 +21,7 @@ from aliyunsdkslb.request.v20140515.DescribeHealthStatusRequest import DescribeHealthStatusRequest from aliyunsdkslb.request.v20140515.DescribeLoadBalancerAttributeRequest import DescribeLoadBalancerAttributeRequest from aliyunsdkslb.request.v20140515.DescribeLoadBalancersRequest import DescribeLoadBalancersRequest +from aliyunsdkecs.request.v20140526.DescribeVpcsRequest import DescribeVpcsRequest from c7n.utils import local_session from c7n.utils import type_schema @@ -69,7 +70,7 @@ def get_request(self, i): request.set_LoadBalancerId(i['LoadBalancerId']) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) if self.data['value'] in jmespath.search(self.listener_protocol_key, data): return False @@ -138,6 +139,41 @@ def get_request(self, i): return i return False +@Slb.filter_registry.register('listener-type') +class AliyunSlbListener(AliyunSlbListenerFilter): + """Filters + + :Example: + + .. code-block:: yaml + + policies: + # 检测您账号下的SLB负载均衡开启HTTPS或HTTP监听视为合规，否则不合规。 + - name: aliyun-slb-listener-type + resource: aliyun.slb + filters: + - type: listener-type + value: ["https", "http"] + """ + schema = type_schema( + 'listener-type', + **{'value': {'type': 'array', 'items': {'type': 'string'}}}) + + def get_request(self, i): + request = DescribeLoadBalancerAttributeRequest() + request.set_accept_format('json') + request.set_LoadBalancerId(i['LoadBalancerId']) + client = local_session( + self.manager.session_factory).client(self.manager.get_model().service) + response = json.loads(client.do_action(request)) + ListenerPortAndProtocal = response.get('ListenerPortsAndProtocal').get('ListenerPortAndProtocal') + if len(ListenerPortAndProtocal) == 0: + return i + for ListenerProtocol in ListenerPortAndProtocal: + if ListenerProtocol in self.data['value']: + return False + return False + @Slb.filter_registry.register('metrics') class SlbMetricsFilter(MetricsFilter): @@ -159,6 +195,8 @@ def get_request(self, r): request.set_accept_format('json') request.set_StartTime(self.start) request.set_Period(self.period) + dimensions = self.get_dimensions(r) + request.set_Dimensions(dimensions) request.set_Namespace(self.namespace) request.set_MetricName(self.metric) return request @@ -210,6 +248,30 @@ def get_request(self, i): return False return i +@Slb.filter_registry.register('vpc-type') +class VpcSlbFilter(AliyunSlbFilter): + """Filters + :Example: + .. code-block:: yaml + + policies: + # 检测您账号下SLB负载均衡实例指定属于哪些VPC, 属于则合规，不属于则"不合规"。 + - name: aliyun-slb-vpc-type + resource: aliyun.slb + filters: + - type: vpc-type + vpcIds: ["111", "222"] + """ + schema = type_schema( + 'vpc-type', + **{'vpcIds': {'type': 'array', 'items': {'type': 'string'}}}) + + def get_request(self, i): + vpcId = i['VpcId'] + if vpcId in self.data['vpcIds']: + return False + return i + @Slb.filter_registry.register('bandwidth') class BandwidthSlbFilter(AliyunSlbFilter): """Filters @@ -233,7 +295,7 @@ def get_request(self, i): request.set_accept_format('json') request.set_LoadBalancerId(i['LoadBalancerId']) response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) if self.data['value'] < data['Bandwidth']: return False @@ -259,23 +321,16 @@ class AclsSlbFilter(AliyunSlbFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - request = DescribeAccessControlListsRequest() - request.set_accept_format('json') - response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") - data = eval(string) - if self.data['value']: - if len(data) == 0: - return False + AddressType = i.get('AddressType', '') + if self.data.get('value', ''): + if AddressType == "Internet": + return i + else: + return None + else: + if AddressType == "Intranet": + return i else: - for obj in data['Acls']['Acl']: - req = DescribeAccessControlListAttributeRequest() - req.set_accept_format('json') - req.set_AclId(obj['AclId']) - res = Session.client(self, service).do_action_with_exception(req) - string = str(res, encoding="utf-8").replace("false", "False") - data2 = eval(string) - if len(data2) == 0: - return False + return None return i diff --git a/tools/c7n_aliyun/c7n_aliyun/resources/vpc.py b/tools/c7n_aliyun/c7n_aliyun/resources/vpc.py index bb8dfc97932..84e1462fbb1 100644 --- a/tools/c7n_aliyun/c7n_aliyun/resources/vpc.py +++ b/tools/c7n_aliyun/c7n_aliyun/resources/vpc.py @@ -133,18 +133,72 @@ class IPPermission(AliyunSgFilter): def get_request(self, sg): service = 'security-group' - request = DescribeSecurityGroupAttributeRequest(); + request = DescribeSecurityGroupAttributeRequest() request.set_SecurityGroupId(sg["SecurityGroupId"]) request.set_Direction("ingress") request.set_accept_format('json') response = Session.client(self, service).do_action_with_exception(request) - string = str(response, encoding="utf-8").replace("false", "False") + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") data = eval(string) for cidr in jmespath.search(self.ip_permissions_key, data): if cidr['SourceCidrIp'] == self.data['value']: + sg['cidr'] = cidr return sg return False +@SecurityGroup.filter_registry.register('source-ports') +class IPPermission(AliyunSgFilter): + + """Filters + :Example: + .. code-block:: yaml + + policies: + # 账号下ECS安全组配置允许所有端口访问视为不合规，否则为合规 + - name: aliyun-sg-ports + resource: aliyun.security-group + filters: + - type: source-ports + SourceCidrIp: "0.0.0.0/0" + PortRange: -1/-1 + """ + + ip_permissions_key = "Permissions.Permission" + schema = type_schema( + 'source-ports', + **{'SourceCidrIp': {'type': 'string'}, + 'PortRange': {'type': 'string'}} + ) + + def get_request(self, sg): + service = 'security-group' + request = DescribeSecurityGroupAttributeRequest(); + request.set_SecurityGroupId(sg["SecurityGroupId"]) + request.set_Direction("ingress") + request.set_accept_format('json') + response = Session.client(self, service).do_action_with_exception(request) + string = str(response, encoding="utf-8").replace("false", "False").replace("true", "True") + data = eval(string) + for ports in jmespath.search(self.ip_permissions_key, data): + if ports['SourceCidrIp'] == self.data['SourceCidrIp']: + if ports['PortRange'] == self.data['PortRange']: + return sg + values = self.data['PortRange'].split('/') + if values[0].replace("”", "") == "-1": + fromPort = 0 + toPort = 65535 + else: + fromPort = int(values[0].replace("”", "")) + toPort = int(values[1].replace("”", "")) + if '/' in ports['PortRange']: + strs = ports['PortRange'].split('/') + port1 = int(strs[0]) + port2 = int(strs[1]) + if (fromPort >= port1 and fromPort <= port2) \ + or (toPort >= port1 and toPort <= port2): + return sg + return False + @SecurityGroup.filter_registry.register('ingress') class IPPermission(SGPermission): @@ -157,7 +211,7 @@ class IPPermission(SGPermission): schema['properties'].update(SGPermissionSchema) def process_self_cidrs(self, perm): - self.process_cidrs(perm, "SourceCidrIp", "Ipv6SourceCidrIp") + return self.process_cidrs(perm, "SourceCidrIp", "Ipv6SourceCidrIp") def securityGroupAttributeRequst(self, sg): requst = DescribeSecurityGroupAttributeRequest(); @@ -185,4 +239,4 @@ def securityGroupAttributeRequst(self, sg): return requst def process_self_cidrs(self, perm): - self.process_cidrs(perm, "DestCidrIp", "Ipv6DestCidrIp") + return self.process_cidrs(perm, "DestCidrIp", "Ipv6DestCidrIp") diff --git a/tools/c7n_aliyun/setup.py b/tools/c7n_aliyun/setup.py index 97ada937f56..3de1453cdea 100644 --- a/tools/c7n_aliyun/setup.py +++ b/tools/c7n_aliyun/setup.py @@ -4,36 +4,36 @@ from setuptools import setup packages = \ -['c7n_aliyun', 'c7n_aliyun.actions', 'c7n_aliyun.filters', 'c7n_aliyun.resources'] + ['c7n_aliyun', 'c7n_aliyun.actions', 'c7n_aliyun.filters', 'c7n_aliyun.resources'] package_data = \ -{'': ['*']} + {'': ['*']} install_requires = \ -['argcomplete (>=1.11.1,<2.0.0)', - 'attrs (>=19.3.0,<20.0.0)', - 'boto3 (>=1.14.8,<2.0.0)', - 'botocore (>=1.17.8,<2.0.0)', - 'c7n (>=0.9.3,<0.10.0)', - 'docutils (>=0.15.2,<0.16.0)', - 'google-api-python-client>=1.7,<2.0', - 'google-auth>=1.11.0,<2.0.0', - 'google-cloud-logging>=1.14,<2.0', - 'google-cloud-monitoring>=0.34.0,<0.35.0', - 'google-cloud-storage>=1.28.1,<2.0.0', - 'importlib-metadata (>=1.6.1,<2.0.0)', - 'jmespath (>=0.10.0,<0.11.0)', - 'jsonschema (>=3.2.0,<4.0.0)', - 'pyrsistent (>=0.16.0,<0.17.0)', - 'python-dateutil (>=2.8.1,<3.0.0)', - 'pyyaml (>=5.3.1,<6.0.0)', - 'ratelimiter>=1.2.0,<2.0.0', - 'retrying>=1.3.3,<2.0.0', - 's3transfer (>=0.3.3,<0.4.0)', - 'six (>=1.15.0,<2.0.0)', - 'tabulate (>=0.8.7,<0.9.0)', - 'urllib3 (>=1.25.9,<2.0.0)', - 'zipp (>=3.1.0,<4.0.0)'] + ['argcomplete (>=1.11.1,<2.0.0)', + 'attrs (>=19.3.0,<20.0.0)', + 'boto3 (>=1.14.8,<2.0.0)', + 'botocore (>=1.17.8,<2.0.0)', + 'c7n (>=0.9.3,<0.10.0)', + 'docutils (>=0.15.2,<0.16.0)', + 'google-api-python-client>=1.7,<2.0', + 'google-auth>=1.11.0,<2.0.0', + 'google-cloud-logging>=1.14,<2.0', + 'google-cloud-monitoring>=0.34.0,<0.35.0', + 'google-cloud-storage>=1.28.1,<2.0.0', + 'importlib-metadata (>=1.6.1,<2.0.0)', + 'jmespath (>=0.10.0,<0.11.0)', + 'jsonschema (>=3.2.0,<4.0.0)', + 'pyrsistent (>=0.16.0,<0.17.0)', + 'python-dateutil (>=2.8.1,<3.0.0)', + 'pyyaml (>=5.3.1,<6.0.0)', + 'ratelimiter>=1.2.0,<2.0.0', + 'retrying>=1.3.3,<2.0.0', + 's3transfer (>=0.3.3,<0.4.0)', + 'six (>=1.15.0,<2.0.0)', + 'tabulate (>=0.8.7,<0.9.0)', + 'urllib3 (>=1.25.9,<2.0.0)', + 'zipp (>=3.1.0,<4.0.0)'] setup_kwargs = { 'name': 'c7n_aliyun', diff --git a/tools/c7n_aliyun/test/aliyun.py b/tools/c7n_aliyun/test/aliyun.py index 9b8ffd1a640..88802fd8818 100644 --- a/tools/c7n_aliyun/test/aliyun.py +++ b/tools/c7n_aliyun/test/aliyun.py @@ -303,4 +303,4 @@ def get_instance_monitor(): # get_slb_listener() # get_instance_monitor() # get_disk() - list_slb() \ No newline at end of file + # list_slb() \ No newline at end of file diff --git a/tools/c7n_azure/c7n_azure/query.py b/tools/c7n_azure/c7n_azure/query.py index 770a6c0f12f..0c876adab21 100644 --- a/tools/c7n_azure/c7n_azure/query.py +++ b/tools/c7n_azure/c7n_azure/query.py @@ -55,9 +55,6 @@ def filter(self, resource_manager, **params): op = getattr(getattr(resource_manager.get_client(), enum_op), list_op) result = op(**params) - for obj in result: - obj.F2CId = obj.id - if isinstance(result, Iterable): return [r.serialize(True) for r in result] elif hasattr(result, 'value'): diff --git a/tools/c7n_huawei/.gitignore b/tools/c7n_huawei/.gitignore index 1e7bb232c6a..8b7f5425938 100644 --- a/tools/c7n_huawei/.gitignore +++ b/tools/c7n_huawei/.gitignore @@ -2,3 +2,4 @@ *py~ __pycache__ *.egg-info +huawei.py \ No newline at end of file diff --git a/tools/c7n_huawei/c7n_huawei/filters/filter.py b/tools/c7n_huawei/c7n_huawei/filters/filter.py index 844e207bc14..5bd8c54a359 100644 --- a/tools/c7n_huawei/c7n_huawei/filters/filter.py +++ b/tools/c7n_huawei/c7n_huawei/filters/filter.py @@ -1,9 +1,7 @@ import datetime -import json from concurrent.futures import as_completed from datetime import timedelta -import jmespath from dateutil.parser import parse from dateutil.tz import tzutc from openstack import utils @@ -15,6 +13,7 @@ from c7n.utils import local_session, chunks from c7n.utils import type_schema + class HuaweiEcsFilter(Filter): schema = None @@ -296,13 +295,12 @@ def process(self, resources, event=None): def process_ports(self, perm): found = None - if perm['port_range_max'] and perm['port_range_min']: + if perm['port_range_max'] is not None and perm['port_range_min'] is not None: FromPort = int(perm['port_range_max']) ToPort = int(perm['port_range_min']) for port in self.ports: if port >= FromPort and port <= ToPort: - found = True - break + return True found = False only_found = False for port in self.only_ports: @@ -404,8 +402,8 @@ def __call__(self, resource): if perm['direction'] != self.direction: continue perm_matches = {} - perm_matches['ports'] = self.process_ports(perm) perm_matches['cidrs'] = self.process_self_cidrs(perm) + perm_matches['ports'] = self.process_ports(perm) perm_match_values = list(filter( lambda x: x is not None, perm_matches.values())) # account for one python behavior any([]) == False, all([]) == True diff --git a/tools/c7n_huawei/c7n_huawei/query.py b/tools/c7n_huawei/c7n_huawei/query.py index 8c84d828340..9342d846987 100644 --- a/tools/c7n_huawei/c7n_huawei/query.py +++ b/tools/c7n_huawei/c7n_huawei/query.py @@ -43,9 +43,12 @@ def filter(self, resource_manager, **params): if extra_args: params.update(extra_args) + offset = 1 + limit = 100 + res = [] + buckets = [] if m.service == 'obs': result = resource_manager.get_request() - buckets = [] if result is None: return buckets for b in result.body.buckets: @@ -53,17 +56,19 @@ def filter(self, resource_manager, **params): buckets.append(b) return buckets else: - request = resource_manager.get_request() - if request: - result = request - else: - return None - if path is None: - return result - res = jmespath.search(path, eval(str(result))) - if res is not None: - for data in res: - data['F2CId'] = data[m.id] + while 1 <= offset: + request = resource_manager.get_request() + request.limit = limit + request.offset = offset + response = jmespath.search(path, eval(str(request).replace('null', 'None').replace('false', 'False').replace('true', 'True'))) + if response is not None: + for data in response: + data['F2CId'] = data[m.id] + res = res + response + if len(response) == limit: + offset += 1 + else: + return res return res def _invoke_client_enum(self, client, request, params, path): diff --git a/tools/c7n_huawei/c7n_huawei/resources/dds.py b/tools/c7n_huawei/c7n_huawei/resources/dds.py index fd23ca0f17b..cca4878cbb3 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/dds.py +++ b/tools/c7n_huawei/c7n_huawei/resources/dds.py @@ -62,12 +62,12 @@ class InternetAccessRedisFilter(HuaweiRedisFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - groups = i['groups'] - if self.data['value']: + groups = i.get('groups', []) + if self.data.get('value', ''): if len(groups) > 0: for group in groups: - if len(group['nodes']) > 0: - if group['nodes']['public_ip']: + if len(group.get('nodes', [])) > 0: + if group.get('nodes', []).get('public_ip', None): return i return None else: @@ -75,8 +75,8 @@ def get_request(self, i): return i else: for group in groups: - if len(group['nodes']) > 0: - if group['nodes']['public_ip']: + if len(group.get('nodes', [])) > 0: + if group.get('nodes', []).get('public_ip', None): return i return None diff --git a/tools/c7n_huawei/c7n_huawei/resources/disk.py b/tools/c7n_huawei/c7n_huawei/resources/disk.py index 50e321dbf18..6309c5cf025 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/disk.py +++ b/tools/c7n_huawei/c7n_huawei/resources/disk.py @@ -65,8 +65,8 @@ class EncryptedDiskFilter(HuaweiDiskFilter): **{'value': {'type': 'boolean'}}) def get_request(self, disk): - encrypted = self.data['value'] - if disk['encrypted'] != encrypted: + encrypted = self.data.get('value', '') + if disk.get('encrypted', '') != encrypted: return False return disk @@ -101,6 +101,6 @@ class HuaweiDiskFilter(HuaweiDiskFilter): schema = type_schema('available') def get_request(self, disk): - if disk['status'] != 'available': + if disk.get('status', '') != 'available': return False return disk \ No newline at end of file diff --git a/tools/c7n_huawei/c7n_huawei/resources/ecs.py b/tools/c7n_huawei/c7n_huawei/resources/ecs.py index 12b5a89c7f7..816e90e5fed 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/ecs.py +++ b/tools/c7n_huawei/c7n_huawei/resources/ecs.py @@ -74,7 +74,7 @@ def get_request(self, i): else: for addrs in list(data.values()): for addr in addrs: - if addr['os_ext_ip_stype'] == 'floating': + if addr.get('os_ext_ip_stype', '') is not None and addr.get('os_ext_ip_stype', '') == 'floating': return i return False @@ -102,7 +102,7 @@ class EcsAgeFilter(HuaweiAgeFilter): def get_resource_date(self, i): # '2020-07-27T05:55:32.000000' - return i['os_srv_us_glaunched_at'] + return i.get('os_srv_us_glaunched_at', '2021-07-27T05:55:32.000000') @Ecs.filter_registry.register('instance-network-type') class InstanceNetworkTypeEcsFilter(HuaweiEcsFilter): @@ -123,8 +123,8 @@ class InstanceNetworkTypeEcsFilter(HuaweiEcsFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == 'vpc': - if i['metadata']['vpc_id'] is not None: + if self.data.get('value', '') == 'vpc': + if i.get('metadata', {}).get('vpc_id', '') is not None: return False return i @@ -154,7 +154,7 @@ def get_request(self, dimensions): listMetricsDimensionDimensionsMetrics= [ MetricsDimension( name="instance_id", - value=dimensions[0]['id'] + value=dimensions[0].get('id', '') ) ] listMetricInfoMetricsbody = [ diff --git a/tools/c7n_huawei/c7n_huawei/resources/eip.py b/tools/c7n_huawei/c7n_huawei/resources/eip.py index c944e79d949..eba3766a6e6 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/eip.py +++ b/tools/c7n_huawei/c7n_huawei/resources/eip.py @@ -62,7 +62,7 @@ class HuaweiEipFilter(HuaweiEipFilter): schema = type_schema('DOWN') def get_request(self, i): - if i['status'] != "DOWN": + if i.get('status', '') != "DOWN": return False return i @@ -85,6 +85,6 @@ class BandwidthEipFilter(HuaweiEipFilter): **{'value': {'type': 'number'}}) def get_request(self, i): - if self.data['value'] < i['bandwidth_size']: + if self.datai.get('value', 0) < i.get('bandwidth_size', 0): return False return i diff --git a/tools/c7n_huawei/c7n_huawei/resources/elb.py b/tools/c7n_huawei/c7n_huawei/resources/elb.py index e80a6db090c..00d009143a7 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/elb.py +++ b/tools/c7n_huawei/c7n_huawei/resources/elb.py @@ -69,7 +69,7 @@ def get_request(self, i): request = ShowListenerRequest() request.listener_id = data response = Session.client(self, service).show_listener(request) - if jmespath.search('protocol', response) == self.data['value']: + if jmespath.search('protocol', response) == self.data.get('value', ''): return False return i @@ -91,7 +91,7 @@ class UnusedElbFilter(HuaweiElbFilter): schema = type_schema('unused') def get_request(self, i): - listeners = i['pools'] + listeners = i.get('pools', []) # elb 查询elb下是否有监听 if len(listeners) > 0: return False @@ -117,10 +117,10 @@ class AddressTypeElbFilter(HuaweiElbFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == 'internet': - if i['vip_address'] is not None: + if self.data.get('value', '') == 'internet': + if i.get('vip_address', '') is not None: return False else: - if i['vip_address'] is None: + if i.get('vip_address', '') is None: return False return i \ No newline at end of file diff --git a/tools/c7n_huawei/c7n_huawei/resources/iam.py b/tools/c7n_huawei/c7n_huawei/resources/iam.py index 41141a93705..9bd1e4fb0b4 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/iam.py +++ b/tools/c7n_huawei/c7n_huawei/resources/iam.py @@ -68,10 +68,10 @@ class HuaweiIamLoginFilter(HuaweiIamFilter): def get_request(self, i): try: request = ShowUserLoginProtectRequest() - request.user_id = i['id'] + request.user_id = i.get('id', '') response = Session.client(self, service).show_user_login_protect(request) i['login_protect'] = jmespath.search('login_protect', eval(str(response))) - if self.data['value'] == i['login_protect']['enabled']: + if self.data.get('value', '') == i.get('login_protect', {}).get('enabled', ''): return False return i except Exception as e: diff --git a/tools/c7n_huawei/c7n_huawei/resources/obs.py b/tools/c7n_huawei/c7n_huawei/resources/obs.py index 911e27cba91..b3c6c16ac93 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/obs.py +++ b/tools/c7n_huawei/c7n_huawei/resources/obs.py @@ -101,12 +101,12 @@ def process_bucket(self, b): # BUCKET_OWNER_FULL_CONTROL 桶或对象所有者拥有完全控制权限。 acl = Session.client(self, service).getBucketAcl(b.name) b['permission'] = acl.body.grants - if self.data['value'] == 'read': + if self.data.get('value', '') == 'read': for obj in acl.body.grants: if obj.grantee.group == 'Everyone': if obj.permission == 'READ' or obj.permission == 'READ_ACP': return b - if self.data['value'] == 'write': + if self.data.get('value', '') == 'write': for obj in acl.body.grants: if obj.grantee.group == 'Everyone': if obj.permission == 'WRITE' or obj.permission == 'WRITE_ACP': diff --git a/tools/c7n_huawei/c7n_huawei/resources/rds.py b/tools/c7n_huawei/c7n_huawei/resources/rds.py index 08024b0c634..a2aac938629 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/rds.py +++ b/tools/c7n_huawei/c7n_huawei/resources/rds.py @@ -61,7 +61,9 @@ class HuaweiRdsFilter(HuaweiRdsFilter): schema = type_schema('internet') def get_request(self, i): - public_ips = i['public_ips'] + if i.get('public_ips', '') is None: + return None + public_ips = i.get('public_ips', '') if len(public_ips) == 0: return None return i @@ -78,7 +80,7 @@ def get_request(self, dimensions): new_dimensions.append( { "name": "rds_cluster_id", - "value": str(dimension['id']) + "value": str(dimension.get('id', '')) }) try: listMetricsDimensionDimensionsMetrics = [] @@ -86,7 +88,7 @@ def get_request(self, dimensions): listMetricsDimensionDimensionsMetrics.append( MetricsDimension( name="rds_cluster_id", - value=str(dimension['id']) + value=str(dimension.get('id', '')) ) ) listMetricInfoMetricsbody = [ @@ -127,8 +129,8 @@ class InternetAccessRdsFilter(HuaweiRdsFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - public_ips = i['public_ips'] - if self.data['value'] == True: + public_ips = i.get('public_ips', '') + if self.data.get('value', '') == True: if len(public_ips) == 0: return None return i @@ -200,7 +202,7 @@ class InstanceNetworkTypeRdsFilter(HuaweiRdsFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if i['vpc_id'] is not None: + if i.get('vpc_id', '') is not None: return False return i @@ -225,6 +227,6 @@ class InstanceNetworkTypeRdsFilter(HuaweiRdsFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if i['charge_info']['charge_mode'] == self.data['value']: + if i['charge_info']['charge_mode'] == self.data.get('value', ''): return False return i \ No newline at end of file diff --git a/tools/c7n_huawei/c7n_huawei/resources/redis.py b/tools/c7n_huawei/c7n_huawei/resources/redis.py index e5e7a50bf4c..43afd210db1 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/redis.py +++ b/tools/c7n_huawei/c7n_huawei/resources/redis.py @@ -63,7 +63,7 @@ class InternetAccessRedisFilter(HuaweiRedisFilter): def get_request(self, i): public_ips = i['publicip_id'] - if self.data['value']: + if self.data.get('value', ''): if public_ips is None or (len(public_ips) == 0 and i['enable_ssl'] == False): return i return False @@ -94,6 +94,6 @@ class NoPasswordAccessRedisFilter(HuaweiRedisFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if i['no_password_access'] == self.data['value']: + if i['no_password_access'] == self.data.get('value', ''): return i return False diff --git a/tools/c7n_huawei/c7n_huawei/resources/securitygroup.py b/tools/c7n_huawei/c7n_huawei/resources/securitygroup.py index 4b139852678..0cf0a88312f 100644 --- a/tools/c7n_huawei/c7n_huawei/resources/securitygroup.py +++ b/tools/c7n_huawei/c7n_huawei/resources/securitygroup.py @@ -81,7 +81,7 @@ class IPPermission(SGPermission): schema['properties'].update(SGPermissionSchema) def process_self_cidrs(self, perm): - self.process_cidrs(perm, "remote_ip_prefix", "remote_ip_prefix") + return self.process_cidrs(perm, "remote_ip_prefix", "remote_ip_prefix") def securityGroupAttributeRequst(self, sg): @@ -103,7 +103,7 @@ def securityGroupAttributeRequst(self, sg): return sg def process_self_cidrs(self, perm): - self.process_cidrs(perm, "DestCidrIp", "Ipv6DestCidrIp") + return self.process_cidrs(perm, "DestCidrIp", "Ipv6DestCidrIp") @SecurityGroup.filter_registry.register('source-cidr-ip') class HuaweiSgSourceCidrIp(HuaweiSgFilter): @@ -128,6 +128,6 @@ class HuaweiSgSourceCidrIp(HuaweiSgFilter): def get_request(self, sg): for cidr in sg[self.ip_permissions_key]: - if cidr['remote_ip_prefix'] == self.data['value']: + if cidr.get('remote_ip_prefix', '') is not None and cidr.get('remote_ip_prefix', '') == self.data.get('value', ''): return sg return False \ No newline at end of file diff --git a/tools/c7n_huawei/test/huawei.py b/tools/c7n_huawei/test/huawei.py index ba9207c4d77..d03e21bfb93 100644 --- a/tools/c7n_huawei/test/huawei.py +++ b/tools/c7n_huawei/test/huawei.py @@ -325,6 +325,6 @@ def list_elb(): # list_sg() # list_rds() # list_cdn() - list_iam() + # list_iam() # list_obs() # list_redis() \ No newline at end of file diff --git a/tools/c7n_mailer/tests/test_misc.py b/tools/c7n_mailer/tests/test_misc.py index fe619d01d87..ef52c7bb9e2 100644 --- a/tools/c7n_mailer/tests/test_misc.py +++ b/tools/c7n_mailer/tests/test_misc.py @@ -36,6 +36,10 @@ def test_mailer_handle(self): https_proxy ] ) + # Clear http proxy + MAILER_CONFIG['http_proxy'] = '' + MAILER_CONFIG['https_proxy'] = '' + config = handle.config_setup(MAILER_CONFIG) def test_sqs_queue_processor(self): mailer_sqs_queue_processor = sqs_queue_processor.MailerSqsQueueProcessor( diff --git a/tools/c7n_openstack/.gitignore b/tools/c7n_openstack/.gitignore new file mode 100644 index 00000000000..1e7bb232c6a --- /dev/null +++ b/tools/c7n_openstack/.gitignore @@ -0,0 +1,4 @@ +*pyc +*py~ +__pycache__ +*.egg-info diff --git a/tools/c7n_openstack/c7n_openstack/__init__.py b/tools/c7n_openstack/c7n_openstack/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/actions/__init__.py b/tools/c7n_openstack/c7n_openstack/actions/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/actions/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/actions/core.py b/tools/c7n_openstack/c7n_openstack/actions/core.py new file mode 100644 index 00000000000..6263705fb9b --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/actions/core.py @@ -0,0 +1,117 @@ +# Copyright 2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from c7n.actions import Action as BaseAction +from c7n.utils import local_session, chunks + + +class Action(BaseAction): + pass + + +class MethodAction(Action): + """Invoke an api call on each resource. + + Quite a number of procedural actions are simply invoking an api + call on a filtered set of resources. The exact handling is mostly + boilerplate at that point following an 80/20 rule. This class is + an encapsulation of the 80%. + """ + + # method we'll be invoking + method_spec = () + + # batch size + chunk_size = 20 + + # implicitly filter resources by state, (attr_name, (valid_enum)) + attr_filter = () + + # error codes that can be safely ignored + ignore_error_codes = () + + permissions = () + method_perm = None + + def validate(self): + if not self.method_spec: + raise NotImplementedError("subclass must define method_spec") + return self + + def filter_resources(self, resources): + rcount = len(resources) + attr_name, valid_enum = self.attr_filter + resources = [r for r in resources if r.get(attr_name) in valid_enum] + if len(resources) != rcount: + self.log.warning( + "policy:%s action:%s implicitly filtered %d resources to %d by attr:%s", + self.manager.ctx.policy.name, + self.type, + rcount, + len(resources), + attr_name, + ) + return resources + + def process(self, resources): + + if self.attr_filter: + resources = self.filter_resources(resources) + model = self.manager.get_model() + session = local_session(self.manager.session_factory) + client = self.get_client(session, model) + for resource_set in chunks(resources, self.chunk_size): + self.process_resource_set(client, model, resource_set) + + def process_resource_set(self, client, model, resources): + result_key = self.method_spec.get('result_key') + annotation_key = self.method_spec.get('annotation_key') + for resource in resources: + requst = self.get_request(resource) + result = self.invoke_api(client, requst) + if result_key and annotation_key: + resource[annotation_key] = result.get(result_key) + + def invoke_api(self, client, requst): + try: + return client.do_action(requst) + except: + raise + + def get_permissions(self): + if self.permissions: + return self.permissions + m = self.manager.resource_type + method = self.method_perm + if not method and 'op' not in self.method_spec: + return () + if not method: + method = self.method_spec['op'] + component = m.component + if '.' in component: + component = component.split('.')[-1] + return ("{}.{}.{}".format( + m.perm_service or m.service, component, method),) + + def get_operation_name(self, model, resource): + return self.method_spec['op'] + + def get_resource_params(self, model, resource): + raise NotImplementedError("subclass responsibility") + + def get_request(self, resource): + raise NotImplementedError("subclass responsibility") + + def get_client(self, session, model): + return session.client(model.service) diff --git a/tools/c7n_openstack/c7n_openstack/actions/cscc.py b/tools/c7n_openstack/c7n_openstack/actions/cscc.py new file mode 100644 index 00000000000..2981090db7f --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/actions/cscc.py @@ -0,0 +1,243 @@ +# Copyright 2018-2019 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime +import hashlib +import json +from urllib.parse import urlparse + +from c7n_openstack.provider import resources as openstack_resources + +from c7n.exceptions import PolicyExecutionError, PolicyValidationError +from c7n.utils import local_session, type_schema +from .core import MethodAction + + +class PostFinding(MethodAction): + """Post finding for matched resources to Cloud Security Command Center. + + + :Example: + + .. code-block:: yaml + + policies: + - name: openstack-instances-with-label + resource: openstack.instance + filters: + - "tag:name": "bad-instance" + actions: + - type: post-finding + org-domain: example.io + category: MEDIUM_INTERNET_SECURITY + + The source for custodian can either be specified inline to the policy, or + custodian can generate one at runtime if it doesn't exist given a org-domain + or org-id. + + Finding updates are not currently supported, due to upstream api issues. + """ + schema = type_schema( + 'post-finding', + **{ + 'source': { + 'type': 'string', + 'description': 'qualified name of source to post to CSCC as'}, + 'org-domain': {'type': 'string'}, + 'org-id': {'type': 'integer'}, + 'category': {'type': 'string'}}) + schema_alias = True + method_spec = {'op': 'create', 'result': 'name', 'annotation_key': 'c7n:Finding'} + + # create throws error if already exists, patch method has bad docs. + ignore_error_codes = (409,) + + CustodianSourceName = 'CloudCustodian' + DefaultCategory = 'Custodian' + Service = 'securitycenter' + ServiceVersion = 'v1beta1' + + _source = None + + # security center permission model is pretty obtuse to correct + permissions = ( + 'securitycenter.findings.list', + 'securitycenter.findings.update', + 'resourcemanager.organizations.get', + 'securitycenter.assetsecuritymarks.update', + 'securitycenter.sources.update', + 'securitycenter.sources.list' + ) + + def validate(self): + if not any([self.data.get(k) for k in ('source', 'org-domain', 'org-id')]): + raise PolicyValidationError( + "policy:%s CSCC post-finding requires one of source, org-domain, org-id" % ( + self.manager.ctx.policy.name)) + + def process(self, resources): + self.initialize_source() + return super(PostFinding, self).process(resources) + + def get_client(self, session, model): + return session.client( + self.Service, self.ServiceVersion, 'organizations.sources.findings') + + def get_resource_params(self, model, resource): + return self.get_finding(resource) + + def initialize_source(self): + # Ideally we'll be given a source, but we'll attempt to auto create it + # if given an org_domain or org_id. + if self._source: + return self._source + elif 'source' in self.data: + self._source = self.data['source'] + return self._source + + session = local_session(self.manager.session_factory) + + # Resolve Organization Id + if 'org-id' in self.data: + org_id = self.data['org-id'] + else: + orgs = session.client('cloudresourcemanager', 'v1', 'organizations') + res = orgs.execute_query( + 'search', {'body': { + 'filter': 'domain:%s' % self.data['org-domain']}}).get( + 'organizations') + if not res: + raise PolicyExecutionError("Could not determine organization id") + org_id = res[0]['name'].rsplit('/', 1)[-1] + + # Resolve Source + client = session.client(self.Service, self.ServiceVersion, 'organizations.sources') + source = None + res = [s for s in + client.execute_query( + 'list', {'parent': 'organizations/{}'.format(org_id)}).get('sources') + if s['displayName'] == self.CustodianSourceName] + if res: + source = res[0]['name'] + + if source is None: + source = client.execute_command( + 'create', + {'parent': 'organizations/{}'.format(org_id), + 'body': { + 'displayName': self.CustodianSourceName, + 'description': 'Cloud Management Rules Engine'}}).get('name') + self.log.info( + "policy:%s resolved cscc source: %s, update policy with this source value", + self.manager.ctx.policy.name, + source) + self._source = source + return self._source + + def get_name(self, r): + """Given an arbitrary resource attempt to resolve back to a qualified name.""" + namer = ResourceNameAdapters[self.manager.resource_type.service] + return namer(r) + + def get_finding(self, resource): + policy = self.manager.ctx.policy + resource_name = self.get_name(resource) + # ideally we could be using shake, but its py3.6+ only + finding_id = hashlib.sha256( + b"%s%s" % ( + policy.name.encode('utf8'), + resource_name.encode('utf8'))).hexdigest()[:32] + + finding = { + 'name': '{}/findings/{}'.format(self._source, finding_id), + 'resourceName': resource_name, + 'state': 'ACTIVE', + 'category': self.data.get('category', self.DefaultCategory), + 'eventTime': datetime.datetime.utcnow().isoformat('T') + 'Z', + 'sourceProperties': { + 'resource_type': self.manager.type, + 'title': policy.data.get('title', policy.name), + 'policy_name': policy.name, + 'policy': json.dumps(policy.data) + } + } + + request = { + 'parent': self._source, + 'findingId': finding_id[:31], + 'body': finding} + return request + + @classmethod + def register_resource(klass, registry, resource_class): + if resource_class.resource_type.service not in ResourceNameAdapters: + return + if 'post-finding' in resource_class.action_registry: + return + resource_class.action_registry.register('post-finding', klass) + + +# CSCC uses its own notion of resource id, if we want our findings on +# a resource to be linked from the asset view we need to post w/ the +# same resource name. If this conceptulization of resource name is +# standard, then we should move these to resource types with +# appropriate hierarchies by service. + + +def name_compute(r): + prefix = urlparse(r['selfLink']).path.strip('/').split('/')[2:][:-1] + return "//compute.googleapis.com/{}/{}".format( + "/".join(prefix), + r['id']) + + +def name_iam(r): + return "//iam.googleapis.com/projects/{}/serviceAccounts/{}".format( + r['projectId'], + r['uniqueId']) + + +def name_resourcemanager(r): + rid = r.get('projectNumber') + if rid is not None: + rtype = 'projects' + else: + rid = r.get('organizationId') + rtype = 'organizations' + return "//cloudresourcemanager.googleapis.com/{}/{}".format( + rtype, rid) + + +def name_container(r): + return "//container.googleapis.com/{}".format( + "/".join(urlparse(r['selfLink']).path.strip('/').split('/')[1:])) + + +def name_storage(r): + return "//storage.googleapis.com/{}".format(r['name']) + + +def name_appengine(r): + return "//appengine.googleapis.com/{}".format(r['name']) + + +ResourceNameAdapters = { + 'appengine': name_appengine, + 'cloudresourcemanager': name_resourcemanager, + 'compute': name_compute, + 'container': name_container, + 'iam': name_iam, + 'storage': name_storage, +} + +openstack_resources.subscribe(PostFinding.register_resource) diff --git a/tools/c7n_openstack/c7n_openstack/client.py b/tools/c7n_openstack/c7n_openstack/client.py new file mode 100644 index 00000000000..6a28cda1466 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/client.py @@ -0,0 +1,40 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2017 The Forseti Security Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import openstack + +log = logging.getLogger('custodian.openstack.client') + + +class Session: + def __init__(self, regionId=None): + self.http_proxy = os.getenv('HTTPS_PROXY') + self.cloud_name = os.getenv('OS_CLOUD_NAME') + if not regionId: + regionId = os.getenv('OS_DEFAULT_REGION') + self.regionId = regionId + + def client(self): + if self.cloud_name: + log.debug(f"Connect to OpenStack cloud {self.cloud_name}") + else: + log.debug(("OpenStack cloud name not set, " + "try to get openstack credential from environment")) + cloud = openstack.connect(cloud=self.cloud_name) + return cloud \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/entry.py b/tools/c7n_openstack/c7n_openstack/entry.py new file mode 100644 index 00000000000..8fce605657a --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/entry.py @@ -0,0 +1,32 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +import logging +from c7n_openstack.resources import ( + project, + flavor, + server, + user, + volume, + image, + network, + router +) + +log = logging.getLogger('custodian.openstack') + +ALL = [ + flavor, + project, + server, + user, + volume, + image, + network, + router +] + + +def initialize_openstack(): + """openstack entry point + """ \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/filters/__init__.py b/tools/c7n_openstack/c7n_openstack/filters/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/filters/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/filters/filter.py b/tools/c7n_openstack/c7n_openstack/filters/filter.py new file mode 100644 index 00000000000..7bd925e3d89 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/filters/filter.py @@ -0,0 +1,340 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +import json + +import jmespath + +from c7n.exceptions import PolicyValidationError +from c7n.filters.core import Filter +from c7n.filters.core import ValueFilter + + +class PolicyFilter: + pass + +class Filter(Filter): + + def validate(self): + return self + + def __call__(self, i): + return i + +class OpenstackFilter(Filter): + + def validate(self): + return self + + def __call__(self, i): + return self.get_request(i) + +class SGPermission(Filter): + """Filter for verifying security group ingress and egress permissions + + All attributes of a security group permission are available as + value filters. + + If multiple attributes are specified the permission must satisfy + all of them. Note that within an attribute match against a list value + of a permission we default to or. + + If a group has any permissions that match all conditions, then it + matches the filter. + + Permissions that match on the group are annotated onto the group and + can subsequently be used by the remove-permission action. + + We have specialized handling for matching `Ports` in ingress/egress + permission From/To range. The following example matches on ingress + rules which allow for a range that includes all of the given ports. + + .. code-block:: yaml + + - type: ingress + Ports: [22, 443, 80] + + As well for verifying that a rule only allows for a specific set of ports + as in the following example. The delta between this and the previous + example is that if the permission allows for any ports not specified here, + then the rule will match. ie. OnlyPorts is a negative assertion match, + it matches when a permission includes ports outside of the specified set. + + .. code-block:: yaml + + - type: ingress + OnlyPorts: [22] + + For simplifying ipranges handling which is specified as a list on a rule + we provide a `Cidr` key which can be used as a value type filter evaluated + against each of the rules. If any iprange cidr match then the permission + matches. + + .. code-block:: yaml + + - type: ingress + IpProtocol: -1 + FromPort: 445 + + We also have specialized handling for matching self-references in + ingress/egress permissions. The following example matches on ingress + rules which allow traffic its own same security group. + + .. code-block:: yaml + + - type: ingress + SelfReference: True + + As well for assertions that a ingress/egress permission only matches + a given set of ports, *note* OnlyPorts is an inverse match. + + .. code-block:: yaml + + - type: egress + OnlyPorts: [22, 443, 80] + + - type: egress + Cidr: + value_type: cidr + op: in + value: x.y.z + + `Cidr` can match ipv4 rules and `CidrV6` can match ipv6 rules. In + this example we are blocking global inbound connections to SSH or + RDP. + + .. code-block:: yaml + + - type: ingress + Ports: [22, 3389] + Cidr: + value: + - "0.0.0.0/0" + - "::/0" + op: in + + `SGReferences` can be used to filter out SG references in rules. + In this example we want to block ingress rules that reference a SG + that is tagged with `Access: Public`. + + .. code-block:: yaml + + - type: ingress + SGReferences: + key: "tag:Access" + value: "Public" + op: equal + + We can also filter SG references based on the VPC that they are + within. In this example we want to ensure that our outbound rules + that reference SGs are only referencing security groups within a + specified VPC. + + .. code-block:: yaml + + - type: egress + SGReferences: + key: 'VpcId' + value: 'vpc-11a1a1aa' + op: equal + + Likewise, we can also filter SG references by their description. + For example, we can prevent egress rules from referencing any + SGs that have a description of "default - DO NOT USE". + + .. code-block:: yaml + + - type: egress + SGReferences: + key: 'Description' + value: 'default - DO NOT USE' + op: equal + + """ + + perm_attrs = { + 'IpProtocol', "Priority", 'Policy'} + filter_attrs = { + 'Cidr', 'CidrV6', 'Ports', 'OnlyPorts', + 'SelfReference', 'Description', 'SGReferences'} + attrs = perm_attrs.union(filter_attrs) + attrs.add('match-operator') + attrs.add('match-operator') + + def validate(self): + delta = set(self.data.keys()).difference(self.attrs) + delta.remove('type') + if delta: + raise PolicyValidationError("Unknown keys %s on %s" % ( + ", ".join(delta), self.manager.data)) + return self + + def process(self, resources, event=None): + self.vfilters = [] + fattrs = list(sorted(self.perm_attrs.intersection(self.data.keys()))) + self.ports = 'Ports' in self.data and self.data['Ports'] or () + self.only_ports = ( + 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) + for f in fattrs: + fv = self.data.get(f) + if isinstance(fv, dict): + fv['key'] = f + else: + fv = {f: fv} + vf = ValueFilter(fv, self.manager) + vf.annotate = False + + self.vfilters.append(vf) + + return super(SGPermission, self).process(resources, event) + + def process_ports(self, perm): + found = None + if perm['remote_ip_prefix'] == '0.0.0.0/0' or perm['remote_ip_prefix'] == '::/0': + return True + if perm['port_range_min'] is None or perm['port_range_max'] is None : + return True + FromPort = int(perm['port_range_min']) + ToPort = int(perm['port_range_max']) + for port in self.ports: + if port >= FromPort and port <= ToPort: + found = True + break + elif FromPort == -1 and ToPort == -1: + found = True + break + else: + found = False + only_found = False + for port in self.only_ports: + if port == FromPort and port == ToPort: + only_found = True + if self.only_ports and not only_found: + found = found is None or found and True or False + if self.only_ports and only_found: + found = False + return found + + + def _process_cidr(self, cidr_key, cidr_type, SourceCidrIp, perm): + found = None + SourceCidrIp = perm.get(SourceCidrIp, "") + if not SourceCidrIp: + return False + SourceCidrIp = {cidr_type: SourceCidrIp} + match_range = self.data[cidr_key] + if isinstance(match_range, dict): + match_range['key'] = cidr_type + else: + match_range = {cidr_type: match_range} + vf = ValueFilter(match_range, self.manager) + vf.annotate = False + found = vf(SourceCidrIp) + if found: + found = True + else: + found = False + return found + + def process_cidrs(self, perm, ipv4Cidr, ipv6Cidr): + found_v6 = found_v4 = None + if 'CidrV6' in self.data: + found_v6 = self._process_cidr('CidrV6', 'CidrIpv6', ipv6Cidr, perm) + if 'Cidr' in self.data: + found_v4 = self._process_cidr('Cidr', 'CidrIp', ipv4Cidr, perm) + match_op = self.data.get('match-operator', 'and') == 'and' and all or any + cidr_match = [k for k in (found_v6, found_v4) if k is not None] + if not cidr_match: + return None + return match_op(cidr_match) + + def process_description(self, perm): + if 'Description' not in self.data: + return None + + d = dict(self.data['Description']) + d['key'] = 'Description' + + vf = ValueFilter(d, self.manager) + vf.annotate = False + + for k in ('Ipv6Ranges', 'IpRanges', 'UserIdGroupPairs', 'PrefixListIds'): + if k not in perm or not perm[k]: + continue + return vf(perm[k][0]) + return False + + def process_self_reference(self, perm, sg_id): + found = None + ref_match = self.data.get('SelfReference') + if ref_match is not None: + found = False + if 'UserIdGroupPairs' in perm and 'SelfReference' in self.data: + self_reference = sg_id in [p['GroupId'] + for p in perm['UserIdGroupPairs']] + if ref_match is False and not self_reference: + found = True + if ref_match is True and self_reference: + found = True + return found + + def process_sg_references(self, perm, owner_id): + sg_refs = self.data.get('SGReferences') + if not sg_refs: + return None + + sg_perm = perm.get('UserIdGroupPairs', []) + if not sg_perm: + return False + + sg_group_ids = [p['GroupId'] for p in sg_perm if p['UserId'] == owner_id] + sg_resources = self.manager.get_resources(sg_group_ids) + vf = ValueFilter(sg_refs, self.manager) + vf.annotate = False + + for sg in sg_resources: + if vf(sg): + return True + return False + + def __call__(self, resource): + result = self.securityGroupAttributeRequst(resource) + matched = [] + match_op = self.data.get('match-operator', 'and') == 'and' and all or any + for perm in jmespath.search(self.ip_permissions_key, result): + perm_matches = {} + perm_matches['ports'] = self.process_ports(perm) + perm_matches['cidrs'] = self.process_self_cidrs(perm) + perm_match_values = list(filter( + lambda x: x is not None, perm_matches.values())) + # account for one python behavior any([]) == False, all([]) == True + if match_op == all and not perm_match_values: + continue + + match = match_op(perm_match_values) + if match: + matched.append(perm) + if matched: + return True + +SGPermissionSchema = { + 'match-operator': {'type': 'string', 'enum': ['or', 'and']}, + 'Ports': {'type': 'array', 'items': {'type': 'integer'}}, + 'OnlyPorts': {'type': 'array', 'items': {'type': 'integer'}}, + 'Policy': {}, + 'IpProtocol': { + 'oneOf': [ + {'enum': ["-1", -1, 'TCP', 'UDP', 'ICMP', 'ICMPV6']}, + {'$ref': '#/definitions/filters/value'} + ] + }, + 'FromPort': {'oneOf': [ + {'$ref': '#/definitions/filters/value'}, + {'type': 'integer'}]}, + 'ToPort': {'oneOf': [ + {'$ref': '#/definitions/filters/value'}, + {'type': 'integer'}]}, + 'IpRanges': {}, + 'Cidr': {}, + 'CidrV6': {}, + 'SGReferences': {} +} \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/filters/labels.py b/tools/c7n_openstack/c7n_openstack/filters/labels.py new file mode 100644 index 00000000000..4d791f5e77b --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/filters/labels.py @@ -0,0 +1,119 @@ +# Copyright 2019 Karol Lassak +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta + +from c7n.filters import Filter, FilterValidationError +from c7n.filters.offhours import Time +from c7n.utils import type_schema + +DEFAULT_TAG = "custodian_status" + + +class LabelActionFilter(Filter): + """Filter resources for label specified future action + + Filters resources by a 'custodian_status' label which specifies a future + date for an action. + + The filter parses the label values looking for an 'op@date' + string. The date is parsed and compared to do today's date, the + filter succeeds if today's date is gte to the target date. + + The optional 'skew' parameter provides for incrementing today's + date a number of days into the future. An example use case might + be sending a final notice email a few days before terminating an + instance, or snapshotting a volume prior to deletion. + + The optional 'skew_hours' parameter provides for incrementing the current + time a number of hours into the future. + + Optionally, the 'tz' parameter can get used to specify the timezone + in which to interpret the clock (default value is 'utc') + + :example: + + .. code-block :: yaml + + policies: + - name: vm-stop-marked + resource: openstack.instance + filters: + - type: marked-for-op + # The default label used is custodian_status + # but that is configurable + label: custodian_status + op: stop + # Another optional label is skew + tz: utc + + + """ + schema = type_schema( + 'marked-for-op', + label={'type': 'string'}, + tz={'type': 'string'}, + skew={'type': 'number', 'minimum': 0}, + skew_hours={'type': 'number', 'minimum': 0}, + op={'type': 'string'}) + + def validate(self): + op = self.data.get('op') + if self.manager and op not in self.manager.action_registry.keys(): + raise FilterValidationError( + "Invalid marked-for-op op:%s in %s" % (op, self.manager.data)) + + tz = Time.get_tz(self.data.get('tz', 'utc')) + if not tz: + raise FilterValidationError( + "Invalid timezone specified '%s' in %s" % ( + self.data.get('tz'), self.manager.data)) + return self + + def process(self, resources, event=None): + self.label = self.data.get('label', DEFAULT_TAG) + self.op = self.data.get('op', 'stop') + self.skew = self.data.get('skew', 0) + self.skew_hours = self.data.get('skew_hours', 0) + self.tz = Time.get_tz(self.data.get('tz', 'utc')) + return super(LabelActionFilter, self).process(resources, event) + + def __call__(self, i): + v = i.get('labels', {}).get(self.label, None) + + if v is None: + return False + if '-' not in v or '_' not in v: + return False + + msg, action, action_date_str = v.rsplit('-', 2) + + if action != self.op: + return False + + try: + action_date = datetime.strptime(action_date_str, '%Y_%m_%d__%H_%M') + except Exception: + self.log.error("could not parse label:%s value:%s on %s" % ( + self.label, v, i['name'])) + return False + + # current_date must match timezones with the parsed date string + if action_date.tzinfo: + action_date = action_date.astimezone(self.tz) + current_date = datetime.now(tz=self.tz) + else: + current_date = datetime.now() + + return current_date >= (action_date - timedelta(days=self.skew, hours=self.skew_hours)) diff --git a/tools/c7n_openstack/c7n_openstack/handler.py b/tools/c7n_openstack/c7n_openstack/handler.py new file mode 100644 index 00000000000..97343cda997 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/handler.py @@ -0,0 +1,56 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import uuid + +# Load resource plugins +from c7n_openstack.entry import initialize_openstack + +from c7n.config import Config +from c7n.loader import PolicyLoader + +initialize_openstack() + +log = logging.getLogger('custodian.openstack.functions') + +logging.getLogger().setLevel(logging.INFO) + + +def run(event, context=None): + # policies file should always be valid in functions so do loading naively + with open('config.json') as f: + policy_config = json.load(f) + + if not policy_config or not policy_config.get('policies'): + log.error('Invalid policy config') + return False + + # setup execution options + options = Config.empty(**policy_config.pop('execution-options', {})) + options.update( + policy_config['policies'][0].get('mode', {}).get('execution-options', {})) + # if output_dir specified use that, otherwise make a temp directory + if not options.output_dir: + options['output_dir'] = get_tmp_output_dir() + + loader = PolicyLoader(options) + policies = loader.load_data(policy_config, 'config.json', validate=False) + if policies: + for p in policies: + log.info("running policy %s", p.name) + p.validate() + p.push(event, context) + return True + + +def get_tmp_output_dir(): + output_dir = '/tmp/' + str(uuid.uuid4()) + if not os.path.exists(output_dir): + try: + os.mkdir(output_dir) + except OSError as error: + log.warning("Unable to make output directory: {}".format(error)) + return output_dir diff --git a/tools/c7n_openstack/c7n_openstack/mu.py b/tools/c7n_openstack/c7n_openstack/mu.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/mu.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/output.py b/tools/c7n_openstack/c7n_openstack/output.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/output.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/policy.py b/tools/c7n_openstack/c7n_openstack/policy.py new file mode 100644 index 00000000000..8157478dffb --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/policy.py @@ -0,0 +1,182 @@ +# Copyright 2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from c7n_openstack import mu +from dateutil.tz import tz + +from c7n.exceptions import PolicyValidationError +from c7n.policy import execution, ServerlessExecutionMode, PullMode +from c7n.utils import local_session, type_schema + +DEFAULT_REGION = 'us-central1' + + +class FunctionMode(ServerlessExecutionMode): + + schema = type_schema( + 'openstack', + **{'execution-options': {'$ref': '#/definitions/basic_dict'}, + 'timeout': {'type': 'string'}, + 'memory-size': {'type': 'integer'}, + 'labels': {'$ref': '#/definitions/string_dict'}, + 'network': {'type': 'string'}, + 'max-instances': {'type': 'integer'}, + 'service-account': {'type': 'string'}, + 'environment': {'$ref': '#/definitions/string_dict'}} + ) + + def __init__(self, policy): + self.policy = policy + self.log = logging.getLogger('custodian.openstack.funcexec') + self.region = policy.options.regions[0] if len(policy.options.regions) else DEFAULT_REGION + + def run(self): + raise NotImplementedError("subclass responsibility") + + def provision(self): + self.log.info("Provisioning policy function %s", self.policy.name) + manager = mu.CloudFunctionManager(self.policy.session_factory, self.region) + return manager.publish(self._get_function()) + + def deprovision(self): + manager = mu.CloudFunctionManager(self.policy.session_factory, self.region) + return manager.remove(self._get_function()) + + def validate(self): + pass + + def _get_function(self): + raise NotImplementedError("subclass responsibility") + + +@execution.register('openstack-periodic') +class PeriodicMode(FunctionMode, PullMode): + """Deploy a policy as a Cloud Functions triggered by Cloud Scheduler + at user defined cron interval via Pub/Sub. + + Default region the function is deployed to is ``us-central1``. In + case you want to change that, use the cli ``--region`` flag. + """ + + schema = type_schema( + 'openstack-periodic', + rinherit=FunctionMode.schema, + required=['schedule'], + **{'trigger-type': {'enum': ['http', 'pubsub']}, + 'tz': {'type': 'string'}, + 'schedule': {'type': 'string'}}) + + def validate(self): + mode = self.policy.data['mode'] + if 'tz' in mode: + error = PolicyValidationError( + "policy:%s openstack-periodic invalid tz:%s" % ( + self.policy.name, mode['tz'])) + # We can't catch all errors statically, our local tz retrieval + # then the form openstack is using, ie. not all the same aliases are + # defined. + tzinfo = tz.gettz(mode['tz']) + if tzinfo is None: + raise error + + def _get_function(self): + events = [mu.PeriodicEvent( + local_session(self.policy.session_factory), + self.policy.data['mode'], + self.region + )] + return mu.PolicyFunction(self.policy, events=events) + + def run(self, event, context): + return PullMode.run(self) + + +@execution.register('openstack-audit') +class ApiAuditMode(FunctionMode): + """Custodian policy execution on openstack api audit logs events. + + Deploys as a Cloud Function triggered by api calls. This allows + you to apply your policies as soon as an api call occurs. Audit + logs creates an event for every api call that occurs in your openstack + account. See `openstack Audit Logs + `_ for more + details. + + Default region the function is deployed to is + ``us-central1``. In case you want to change that, use the cli + ``--region`` flag. + """ + + schema = type_schema( + 'openstack-audit', + methods={'type': 'array', 'items': {'type': 'string'}}, + required=['methods'], + rinherit=FunctionMode.schema) + + def resolve_resources(self, event): + """Resolve a openstack resource from its audit trail metadata. + """ + if self.policy.resource_manager.resource_type.get_requires_event: + return [self.policy.resource_manager.get_resource(event)] + resource_info = event.get('resource') + if resource_info is None or 'labels' not in resource_info: + self.policy.log.warning("Could not find resource information in event") + return + # copy resource name, the api doesn't like resource ids, just names. + if 'resourceName' in event['protoPayload']: + resource_info['labels']['resourceName'] = event['protoPayload']['resourceName'] + + resource = self.policy.resource_manager.get_resource(resource_info['labels']) + return [resource] + + def _get_function(self): + events = [mu.ApiSubscriber( + local_session(self.policy.session_factory), + self.policy.data['mode'])] + return mu.PolicyFunction(self.policy, events=events) + + def validate(self): + if not self.policy.resource_manager.resource_type.get: + raise PolicyValidationError( + "Resource:%s does not implement retrieval method" % ( + self.policy.resource_type)) + + def run(self, event, context): + """Execute a openstack serverless model""" + from c7n.actions import EventAction + + resources = self.resolve_resources(event) + if not resources: + return + + resources = self.policy.resource_manager.filter_resources( + resources, event) + + self.policy.log.info("Filtered resources %d" % len(resources)) + + if not resources: + return + + self.policy.ctx.metrics.put_metric( + 'ResourceCount', len(resources), 'Count', Scope="Policy", + buffer=False) + + for action in self.policy.resource_manager.actions: + if isinstance(action, EventAction): + action.process(resources, event) + else: + action.process(resources) + + return resources diff --git a/tools/c7n_openstack/c7n_openstack/provider.py b/tools/c7n_openstack/c7n_openstack/provider.py new file mode 100644 index 00000000000..f46a3013875 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/provider.py @@ -0,0 +1,34 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +from c7n.registry import PluginRegistry +from c7n.provider import Provider, clouds + +from .resources.resource_map import ResourceMap +from .client import Session + +import logging + +log = logging.getLogger('custodian.openstack') + + +@clouds.register('openstack') +class OpenStack(Provider): + + display_name = 'openstack' + resource_prefix = 'openstack' + resources = PluginRegistry('%s.resources' % resource_prefix) + resource_map = ResourceMap + + def initialize(self, options): + return options + + def initialize_policies(self, policy_collection, options): + return policy_collection + + def get_session_factory(self, options): + """Get a credential/session factory for api usage.""" + return Session + + +resources = OpenStack.resources \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/query.py b/tools/c7n_openstack/c7n_openstack/query.py new file mode 100644 index 00000000000..613684f808f --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/query.py @@ -0,0 +1,113 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +import logging + +from c7n.actions import ActionRegistry +from c7n.filters import FilterRegistry +from c7n.manager import ResourceManager +from c7n.query import sources +from c7n.utils import local_session + +log = logging.getLogger('custodian.openstack.query') + + +class ResourceQuery: + def __init__(self, session_factory): + self.session_factory = session_factory + + def filter(self, resource_manager, **params): + m = resource_manager.resource_type + session = local_session(self.session_factory) + client = session.client() + + enum_op, extra_args = m.enum_spec + if extra_args: + params.update(extra_args) + return self._invoke_client_enum(client, enum_op, params) + + def _invoke_client_enum(self, client, enum_op, params): + res = getattr(client, enum_op)(**params) + return res + + +@sources.register('describe-openstack') +class DescribeSource: + def __init__(self, manager): + self.manager = manager + self.query = ResourceQuery(manager.session_factory) + + def get_resources(self, query): + if query is None: + query = {} + return self.query.filter(self.manager, **query) + + def get_permissions(self): + return () + + def augment(self, resources): + return resources + + +class QueryMeta(type): + """metaclass to have consistent action/filter registry for new resources""" + def __new__(cls, name, parents, attrs): + if 'filter_registry' not in attrs: + attrs['filter_registry'] = FilterRegistry( + '%s.filters' % name.lower()) + if 'action_registry' not in attrs: + attrs['action_registry'] = ActionRegistry( + '%s.actions' % name.lower()) + + return super(QueryMeta, cls).__new__(cls, name, parents, attrs) + + +class QueryResourceManager(ResourceManager, metaclass=QueryMeta): + def __init__(self, data, options): + super(QueryResourceManager, self).__init__(data, options) + self.source = self.get_source(self.source_type) + + def get_permissions(self): + return () + + def get_source(self, source_type): + return sources.get(source_type)(self) + + def get_client(self): + client = local_session(self.session_factory).client() + return client + + def get_model(self): + return self.resource_type + + def get_cache_key(self, query): + return {'source_type': self.source_type, 'query': query} + + @property + def source_type(self): + return self.data.get('source', 'describe-openstack') + + def get_resource_query(self): + if 'query' in self.data: + return {'filter': self.data.get('query')} + + def resources(self, query=None): + q = query or self.get_resource_query() + key = self.get_cache_key(q) + resources = self.augment(self.source.get_resources(q)) + self._cache.save(key, resources) + return self.filter_resources(resources) + + def augment(self, resources): + return resources + + +class TypeMeta(type): + def __repr__(cls): + return "" % ( + cls.group, + cls.version) + + +class TypeInfo(metaclass=TypeMeta): + enum_spec = () \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/__init__.py b/tools/c7n_openstack/c7n_openstack/resources/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/flavor.py b/tools/c7n_openstack/c7n_openstack/resources/flavor.py new file mode 100644 index 00000000000..6b18ea92d17 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/flavor.py @@ -0,0 +1,69 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo + + +@resources.register('flavor') +class Flavor(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_flavors', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'vcpus', 'ram', 'disk'] + +@Flavor.filter_registry.register('system') +class FlavorFilter(Filter): + """Filters Flavors based on their system + :example: + .. code-block:: yaml + policies: + - name: openstack-flavor + resource: openstack.flavor + filters: + - type: system + is_public: true + ram: 512 + vcpus: 1 + disk: 1 + """ + + schema = type_schema( + 'system', + is_public={'type': 'boolean'}, + ram={'type': 'number'}, + vcpus={'type': 'number'}, + disk={'type': 'number'}, + ) + + def _match_(self, flavor, is_public, ram, vcpus, disk): + if is_public: + if is_public != flavor.is_public: + return True + else: + if is_public == flavor.is_public: + return True + if ram: + if flavor.ram > ram: + return True + if vcpus: + if flavor.vcpus > vcpus: + return True + if disk: + if flavor.disk > disk: + return True + return False + + def process(self, resources, event=None): + results = [] + is_public = self.data.get('is_public', None) + ram = self.data.get('ram', None) + vcpus = self.data.get('vcpus', None) + disk = self.data.get('disk', None) + for flavor in resources: + if self._match_(flavor, is_public, ram, vcpus, disk): + results.append(flavor) + return results \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/image.py b/tools/c7n_openstack/c7n_openstack/resources/image.py new file mode 100644 index 00000000000..7dfbe6981bf --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/image.py @@ -0,0 +1,62 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n.filters import Filter +from c7n.utils import local_session +from c7n.utils import type_schema +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo + + +@resources.register('image') +class Image(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_images', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'status', 'visibility'] + +@Image.filter_registry.register('status') +class StatusFilter(Filter): + """Filters Images based on their status + :example: + .. code-block:: yaml + policies: + - name: openstack-image + resource: openstack.image + filters: + - not: + - type: status + image_name: centos + visibility: private + status: active + """ + schema = type_schema( + 'status', + image_name={'type': 'string'}, + visibility={'type': 'string'}, + status={'type': 'string'} + ) + + def process(self, resources, event=None): + results = [] + image_name = self.data.get('image_name', None) + visibility = self.data.get('visibility', None) + status = self.data.get('status', None) + + for image in resources: + matched = True + if not image: + if status == "absent": + results.append(image) + continue + if image_name is not None and image_name != image.name: + matched = False + if visibility is not None and visibility != image.visibility: + matched = False + if status is not None and status != image.status: + matched = False + if matched: + results.append(image) + return results + diff --git a/tools/c7n_openstack/c7n_openstack/resources/network.py b/tools/c7n_openstack/c7n_openstack/resources/network.py new file mode 100644 index 00000000000..b3c414ff7ce --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/network.py @@ -0,0 +1,59 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo +from c7n.filters import Filter +from c7n.utils import type_schema + +@resources.register('network') +class Network(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_networks', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'status', 'shared'] + +@Network.filter_registry.register('system') +class NetworkFilter(Filter): + """Filters Networks based on their system + :example: + .. code-block:: yaml + policies: + - name: openstack-network + resource: openstack.network + filters: + - not: + - type: system + status: ACTIVE + shared: false + port_security_enabled: true + """ + schema = type_schema( + 'system', + status={'type': 'string'}, + shared={'type': 'boolean'}, + port_security_enabled={'type': 'boolean'}, + ) + + def _match_(self, network, status, shared, port_security_enabled): + if status: + if status != network.status: + return True + if shared: + if shared != network.shared: + return True + if port_security_enabled: + if port_security_enabled != network.port_security_enabled: + return True + return False + + def process(self, resources, event=None): + results = [] + status = self.data.get('status', None) + shared = self.data.get('shared', None) + port_security_enabled = self.data.get('port_security_enabled', None) + for network in resources: + if self._match_(network, status, shared, port_security_enabled): + results.append(network) + return results diff --git a/tools/c7n_openstack/c7n_openstack/resources/project.py b/tools/c7n_openstack/c7n_openstack/resources/project.py new file mode 100644 index 00000000000..ea8eaebb640 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/project.py @@ -0,0 +1,52 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n_openstack.query import QueryResourceManager, TypeInfo +from c7n_openstack.provider import resources +from c7n.utils import local_session +from c7n.utils import type_schema +from c7n.filters import Filter + + +@resources.register('project') +class Project(QueryResourceManager): + class resource_type(TypeInfo): + id = 'id' + name = 'name' + enum_spec = ('list_projects', None) + default_report_fields = ['id', 'name'] + +@Project.filter_registry.register('user') +class UserFilter(Filter): + """Filters Projects based on their user + :example: + .. code-block:: yaml + policies: + - name: demo + resource: openstack.project + filters: + - type: user + system_scope: true + """ + schema = type_schema( + 'user', + system_scope={'type': 'boolean'}, + ) + + def process(self, resources, event=None): + results = [] + system_scope = self.data.get('system_scope', None) + openstack = local_session(self.manager.session_factory).client() + for project in resources: + users = openstack.list_users() + params = [] + for user in users: + if system_scope: + if user.default_project_id != project.id: + params.append(user) + else: + if user.default_project_id == project.id: + params.append(user) + if len(params) == 0: + results.append(project) + return results \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/resource_map.py b/tools/c7n_openstack/c7n_openstack/resources/resource_map.py new file mode 100644 index 00000000000..d2bb97fb068 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/resource_map.py @@ -0,0 +1,13 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +ResourceMap = { + "openstack.project": "c7n_openstack.resources.project.Project", + "openstack.server": "c7n_openstack.resources.server.Server", + "openstack.flavor": "c7n_openstack.resources.server.Flavor", + "openstack.user": "c7n_openstack.resources.user.User", + "openstack.volume": "c7n_openstack.resources.volume.Volume", + "openstack.image": "c7n_openstack.resources.image.Image", + "openstack.network": "c7n_openstack.resources.network.Network", + "openstack.security-groups": "c7n_openstack.resources.security-groups.SecurityGroups", + "openstack.router": "c7n_openstack.resources.router.Router", +} \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/router.py b/tools/c7n_openstack/c7n_openstack/resources/router.py new file mode 100644 index 00000000000..5e6ad93a1f4 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/router.py @@ -0,0 +1,54 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n.utils import type_schema +from c7n_openstack.filters.filter import Filter +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo + + +@resources.register('router') +class Router(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_routers', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'status', 'ha'] + +@Router.filter_registry.register('system') +class RouterFilter(Filter): + """Filters Routers based on their system + :example: + .. code-block:: yaml + policies: + - name: openstack-router + resource: openstack.router + filters: + - not: + - type: system + status: ACTIVE + ha: true + """ + schema = type_schema( + 'system', + status={'type': 'string'}, + ha={'type': 'boolean'}, + ) + + def _match_(self, router, status, ha): + if ha: + if ha != router.ha: + return True + if status: + if status != router.status: + return True + return False + + def process(self, resources, event=None): + results = [] + status = self.data.get('status', None) + ha = self.data.get('ha', None) + for router in resources: + if self._match_(router, status, ha): + results.append(router) + return results diff --git a/tools/c7n_openstack/c7n_openstack/resources/security-groups.py b/tools/c7n_openstack/c7n_openstack/resources/security-groups.py new file mode 100644 index 00000000000..aebb23c8572 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/security-groups.py @@ -0,0 +1,94 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import jmespath + +from c7n.utils import type_schema +from c7n_openstack.filters.filter import SGPermission +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo +from c7n_openstack.filters.filter import SGPermissionSchema, OpenstackFilter + + +@resources.register('security-groups') +class SecurityGroups(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_security_groups', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name'] + +@SecurityGroups.filter_registry.register('ingress') +class IPIngressPermission(SGPermission): + """Filters SecurityGroups based on their system + :example: + .. code-block:: yaml + policies: + # 扫描开放以下高危端口的安全组： + - name: openstack-security-groups + resource: openstack.security-groups + filters: + - or: + - type: ingress + IpProtocol: "-1" + Ports: [20,21,22,25,80,773,765, 1733,1737,3306,3389,7333,5732,5500] + Cidr: "0.0.0.0/0" + - type: ingress + IpProtocol: "-1" + Ports: [20,21,22,25,80,773,765, 1733,1737,3306,3389,7333,5732,5500] + CidrV6: "::/0" + """ + ip_permissions_key = "security_group_rules" + schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': {'type': {'enum': ['ingress']}}, + 'required': ['type']} + schema['properties'].update(SGPermissionSchema) + + def process_self_cidrs(self, perm): + return self.process_cidrs(perm, 'remote_ip_prefix', 'remote_ip_prefix') + + def securityGroupAttributeRequst(self, sg): + self.direction = 'Ingress' + return sg + +@SecurityGroups.filter_registry.register('egress') +class IPEgressPermission(SGPermission): + schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': {'type': {'enum': ['egress']}}, + 'required': ['type']} + schema['properties'].update(SGPermissionSchema) + + def securityGroupAttributeRequst(self, sg): + self.direction = 'Egress' + return sg + +@SecurityGroups.filter_registry.register('source-cidr-ip') +class SourceCidrIp(OpenstackFilter): + + """Filters + :Example: + .. code-block:: yaml + + policies: + # 账号下安全组配置不为“0.0.0.0/0”，视为“合规”。 + - name: openstack-sg-source-cidr-ip + resource: openstack.security-groups + filters: + - type: source-cidr-ip + value: "0.0.0.0/0" + """ + + ip_permissions_key = "security_group_rules" + schema = type_schema( + 'source-cidr-ip', + **{'value': {'type': 'string'}}) + + def get_request(self, sg): + for cidr in jmespath.search(self.ip_permissions_key, sg): + if cidr['remote_ip_prefix'] == self.data['value']: + return sg + return False diff --git a/tools/c7n_openstack/c7n_openstack/resources/server.py b/tools/c7n_openstack/c7n_openstack/resources/server.py new file mode 100644 index 00000000000..c3f220cf6f9 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/server.py @@ -0,0 +1,232 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n_openstack.query import QueryResourceManager, TypeInfo +from c7n_openstack.provider import resources +from c7n.utils import local_session +from c7n.utils import type_schema +from c7n.filters import Filter +from c7n.filters import AgeFilter + + +@resources.register('server') +class Server(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_servers', None) + id = 'id' + name = 'name' + + set_server_metadata = "set_server_metadata" + delete_server_metadata = "delete_server_metadata" + add_server_tag = "add_server_tag" + set_server_tag = "set_server_tag" + delete_server_tag = "delete_server_tag" + + default_report_fields = ['id', 'name', 'status', 'tenant_id'] + + +@Server.filter_registry.register('image') +class ImageFilter(Filter): + """Filters Servers based on their image attributes + :example: + .. code-block:: yaml + policies: + - name: openstack-image + resource: openstack.server + filters: + - not: + - type: image + image_name: centos + visibility: private + status: active + """ + schema = type_schema( + 'image', + image_name={'type': 'string'}, + visibility={'type': 'string'}, + status={'type': 'string'}) + + def process(self, resources, event=None): + results = [] + client = local_session(self.manager.session_factory).client() + image_name = self.data.get('image_name', None) + visibility = self.data.get('visibility', None) + status = self.data.get('status', None) + + images = client.list_images() + for r in resources: + image = find_object_by_property(images, 'id', r.image.id) + r.image = image + matched = True + if not image: + if status == "absent": + results.append(r) + continue + if image_name is not None and image_name != image.name: + matched = False + if visibility is not None and visibility != image.visibility: + matched = False + if status is not None and status != image.status: + matched = False + if matched: + results.append(r) + return results + + +@Server.filter_registry.register('flavor') +class FlavorFilter(Filter): + """Filters Servers based on their flavor attributes + :example: + .. code-block:: yaml + policies: + - name: openstack-server-flavor + resource: openstack.server + filters: + - type: flavor + is_public: true + ram: 512 + vcpus: 1 + disk: 1 + """ + schema = type_schema( + 'flavor', + flavor_name={'type': 'string'}, + flavor_id={'type': 'string'}, + vcpus={'type': 'integer'}, + ram={'type': 'integer'}, + swap={'type': 'integer'}, + disk={'type': 'integer'}, + ephemeral={'type': 'integer'}, + is_public={'type': 'boolean'}, + ) + + def server_match_flavor(self, server, flavor_name, flavor_id, + vcpus, ram, disk, ephemeral, is_public): + openstack = local_session(self.manager.session_factory).client() + server_flavor_name = server.flavor.original_name + flavor = openstack.get_flavor(server_flavor_name) + if not flavor: + return False + if flavor_name and flavor.name != flavor_name: + return False + if flavor_id and flavor.id != flavor_id: + return False + if vcpus and flavor.vcpus > int(vcpus): + return False + if ram and flavor.ram > int(ram): + return False + if disk and flavor.disk > int(disk): + return False + if ephemeral and flavor.ephemeral != int(ephemeral): + return False + if is_public is not None and flavor.is_public != is_public: + return False + return True + + def process(self, resources, event=None): + results = [] + flavor_name = self.data.get('flavor_name', None) + flavor_id = self.data.get('flavor_id', None) + vcpus = self.data.get('vcpus', None) + ram = self.data.get('ram', None) + disk = self.data.get('disk', None) + ephemeral = self.data.get('ephemeral', None) + is_public = self.data.get('is_public', None) + for server in resources: + if self.server_match_flavor(server, flavor_name, flavor_id, + vcpus, ram, disk, ephemeral, + is_public): + results.append(server) + return results + + +@Server.filter_registry.register('age') +class AgeFilter(AgeFilter): + + date_attribute = "launched_at" + + schema = type_schema( + 'age', + op={'$ref': '#/definitions/filters_common/comparison_operators'}, + days={'type': 'number'}, + hours={'type': 'number'}, + minutes={'type': 'number'}) + + def get_resource_data(self, i): + if i.get("launched_at"): + return i.get("launched_at") + return i.get("created_at") + + +@Server.filter_registry.register('tags') +class TagsFilter(Filter): + """Filters Servers based on their tags + :example: + .. code-block:: yaml + policies: + - name: openstack-server-tags + resource: openstack.server + filters: + - type: tags + tags: + - key: a + value: b + """ + tags_definition = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'key': {'type': 'string'}, + 'value': {'type': 'string'} + }, + 'required': ['key', 'value'], + } + } + schema = type_schema( + 'tags', + tags=tags_definition, + op={'type': 'string', 'enum': ['any', 'all']}, + ) + + def match_any_tags(self, server, tags): + for t in tags: + str_tag = "%s=%s" % (t.get('key'), t.get('value')) + if str_tag in server.tags: + return True + return False + + def match_all_tags(self, server, tags): + for t in tags: + str_tag = "%s=%s" % (t.get('key'), t.get('value')) + if str_tag not in server.tags: + return False + return True + + def process(self, resources, event=None): + results = [] + tags = self.data.get('tags', []) + op = self.data.get('op', 'all') + match_fn = { + 'any': self.match_any_tags, + 'all': self.match_all_tags + } + for server in resources: + if match_fn[op](server, tags): + results.append(server) + return results + + +def find_object_by_property(collection, k, v): + result = [] + for d in collection: + if hasattr(d, k): + value = getattr(d, k) + else: + value = d.get(k) + if (v is None and value is None) or value == v: + result.append(d) + if not result: + return None + assert(len(result) == 1) + return result[0] \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/user.py b/tools/c7n_openstack/c7n_openstack/resources/user.py new file mode 100644 index 00000000000..029664cbf15 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/user.py @@ -0,0 +1,108 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n_openstack.query import QueryResourceManager, TypeInfo +from c7n_openstack.provider import resources +from c7n.utils import local_session +from c7n.utils import type_schema +from c7n.filters import Filter + + +@resources.register('user') +class User(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_users', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'enabled', 'description'] + + +@User.filter_registry.register('project') +class ProjectFilter(Filter): + """Filters Users based on their project + :example: + .. code-block:: yaml + policies: + - name: demo + resource: openstack.user + filters: + - type: project + default_project_id: '' + """ + schema = type_schema( + 'project', + default_project_id={'type': 'string'}, + ) + def process(self, resources, event=None): + results = [] + default_project_id = self.data.get('default_project_id', None) + for user in resources: + if default_project_id != '' and user.default_project_id != default_project_id: + results.append(user) + elif default_project_id == '' and user.default_project_id is None: + results.append(user) + return results + +@User.filter_registry.register('role') +class RoleFilter(Filter): + """Filters Users based on their role + :example: + .. code-block:: yaml + policies: + - name: demo + resource: openstack.user + filters: + - type: role + role_name: admin + system_scope: true + """ + schema = type_schema( + 'role', + role_name={'type': 'string'}, + role_id={'type': 'string'}, + project_name={'type': 'string'}, + project_id={'type': 'string'}, + system_scope={'type': 'boolean'}, + ) + + def user_match_role(self, assignments, user_id, + role_id, project_id, system_scope): + for p in assignments: + if user_id and p.get('user', '') != user_id: + continue + if system_scope and p.get('project'): + continue + if project_id and p.get('project', '') != project_id: + continue + if role_id and p.id != role_id: + continue + return True + return False + + def process(self, resources, event=None): + results = [] + openstack = local_session(self.manager.session_factory).client() + role_name = self.data.get('role_name', None) + role_id = self.data.get('role_id', None) + project_name = self.data.get('project_name', None) + project_id = self.data.get('project_id', None) + system_scope = self.data.get('system_scope', False) + if not role_id and role_name: + role = openstack.get_role(role_name) + if role: + role_id = role.id + else: + raise ValueError(f"Role {role_name} doesn't exists") + if not project_id and project_name: + project = openstack.get_project(project_name) + if project: + project_id = project.id + else: + raise ValueError(f"Project {project_name} doesn't exists") + assignments = openstack.list_role_assignments() + for user in resources: + user_id = user.id + if self.user_match_role(assignments, user_id, role_id, + project_id, system_scope): + results.append(user) + return results \ No newline at end of file diff --git a/tools/c7n_openstack/c7n_openstack/resources/volume.py b/tools/c7n_openstack/c7n_openstack/resources/volume.py new file mode 100644 index 00000000000..eb43f43a628 --- /dev/null +++ b/tools/c7n_openstack/c7n_openstack/resources/volume.py @@ -0,0 +1,61 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_openstack.provider import resources +from c7n_openstack.query import QueryResourceManager, TypeInfo + + +@resources.register('volume') +class Volume(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list_volumes', None) + id = 'id' + name = 'name' + default_report_fields = ['id', 'name', 'status', 'size'] + +@Volume.filter_registry.register('status') +class StatusFilter(Filter): + """Filters Volumes based on their status + :example: + .. code-block:: yaml + policies: + - name: openstack-volume + resource: openstack.volume + filters: + - type: status + volume_status: 'in-use' + system_scope: true + """ + schema = type_schema( + 'status', + is_encrypted={'type': 'boolean'}, + volume_status={'type': 'string'}, + system_scope={'type': 'boolean'}, + ) + + def _match_(self, volume, is_encrypted, volume_status, system_scope): + if is_encrypted: + if is_encrypted != volume.is_encrypted: + return True + if volume_status: + if volume_status != volume.status: + return True + if system_scope: + if len(volume.attachments) == 0: + return True + else: + if len(volume.attachments) > 0: + return True + return False + + def process(self, resources, event=None): + results = [] + is_encrypted = self.data.get('is_encrypted', None) + volume_status = self.data.get('volume_status', None) + system_scope = self.data.get('system_scope', None) + for volume in resources: + if self._match_(volume, is_encrypted, volume_status, system_scope): + results.append(volume) + return results diff --git a/tools/c7n_openstack/pyproject.toml b/tools/c7n_openstack/pyproject.toml new file mode 100644 index 00000000000..85f93b19dde --- /dev/null +++ b/tools/c7n_openstack/pyproject.toml @@ -0,0 +1,29 @@ +[tool.poetry] +name = "c7n_openstack" +version = "0.1.0" +description = "Cloud Custodian - OpenStack Provider" +readme = "readme.md" +authors = ["Cloud Custodian Project"] +homepage = "https://cloudcustodian.io" +repository = "https://github.com/cloud-custodian/cloud-custodian" +documentation = "https://cloudcustodian.io/docs/" +license = "Apache-2.0" +classifiers = [ + "Topic :: System :: Systems Administration", + "Topic :: System :: Distributed Computing" +] + +[tool.poetry.dependencies] +python = "^3.7" +openstacksdk = "^0.52.0" +cryptography = "^2.9.2" + +[tool.poetry.dev-dependencies] +c7n = {path = "../..", develop = true} +pytest = "~6.0.0" +vcrpy = "^4.0.2" + + +[build-system] +requires = ["poetry>=0.12", "setuptools"] +build-backend = "poetry.masonry.api" \ No newline at end of file diff --git a/tools/c7n_openstack/readme.md b/tools/c7n_openstack/readme.md new file mode 100644 index 00000000000..578a85b72d1 --- /dev/null +++ b/tools/c7n_openstack/readme.md @@ -0,0 +1,115 @@ +# Custodian OpenStack Support + +Work in Progress - Not Ready For Use. + +## Quick Start + +### Installation + +``` +pip install c7n_openstack +``` + +### OpenStack Environment Configration + +C7N will find cloud config for as few as 1 cloud and as many as you want to put in a config file. +It will read environment variables and config files, and it also contains some vendor specific default +values so that you don't have to know extra info to use OpenStack: + +* If you have a config file, you will get the clouds listed in it +* If you have environment variables, you will get a cloud named envvars +* If you have neither, you will get a cloud named defaults with base defaults + +Create a clouds.yml file: + +```yaml +clouds: + demo: + region_name: RegionOne + auth: + username: 'admin' + password: XXXXXXX + project_name: 'admin' + domain_name: 'Default' + auth_url: 'https://montytaylor-sjc.openstack.blueboxgrid.com:5001/v2.0' +``` + +Please note: c7n will look for a file called `clouds.yaml` in the following locations: + +* Current Directory +* ~/.config/openstack +* /etc/openstack + +More information at [https://pypi.org/project/os-client-config](https://pypi.org/project/os-client-config) + +### Create a c7n policy yaml file as follows: + +```yaml +policies: +- name: demo + resource: openstack.flavor + filters: + - type: value + key: vcpus + value: 1 + op: gt +``` + +### Run c7n and report the matched resources: + +```sh +mkdir -p output +custodian run demo.yaml -s output +custodian report demo.yaml -s output --format grid +``` + +## Examples + +filter examples: + +```yaml +policies: +- name: test-flavor + resource: openstack.flavor + filters: + - type: value + key: vcpus + value: 1 + op: gt +- name: test-project + resource: openstack.project + filters: [] +- name: test-server-image + resource: openstack.server + filters: + - type: image + image_name: cirros-0.5.1 +- name: test-user + resource: openstack.user + filters: + - type: role + project_name: demo + role_name: _member_ + system_scope: false +- name: test-server-flavor + resource: openstack.server + filters: + - type: flavor + vcpus: 1 +- name: test-server-age + resource: openstack.server + filters: + - type: age + op: lt + days: 1 +- name: test-server-tags + resource: openstack.server + filters: + - type: tags + tags: + - key: a + value: a + - key: b + value: c + op: any +``` diff --git a/tools/c7n_openstack/requirements.txt b/tools/c7n_openstack/requirements.txt new file mode 100644 index 00000000000..0d049ac38dd --- /dev/null +++ b/tools/c7n_openstack/requirements.txt @@ -0,0 +1,3 @@ +openstacksdk==0.52.0; +python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" +keystoneauth1 <= 3.4.0 and keystoneauth1 > = 3.0.0 \ No newline at end of file diff --git a/tools/c7n_openstack/setup.py b/tools/c7n_openstack/setup.py new file mode 100644 index 00000000000..fc5d75a5ab1 --- /dev/null +++ b/tools/c7n_openstack/setup.py @@ -0,0 +1,36 @@ +# Automatically generated from poetry/pyproject.toml +# flake8: noqa +# -*- coding: utf-8 -*- +from setuptools import setup + +packages = [ + 'c7n_openstack', + 'c7n_openstack.resources' +] + +package_data = {'': ['*']} + +install_requires = \ + ['openstacksdk (>=0.52.0)', + 'c7n (>=0.9.8,<0.10.0)'] + + +setup_kwargs = { + 'name': 'c7n-openstack', + 'version': '0.0.1', + 'description': 'Cloud Custodian - OpenStack Provider', + 'long_description': '# Custodian OpenStack Support', + 'long_description_content_type': 'text/markdown', + 'author': 'Cloud Custodian Project', + 'author_email': None, + 'maintainer': None, + 'maintainer_email': None, + 'url': 'https://cloudcustodian.io', + 'packages': packages, + 'package_data': package_data, + 'install_requires': install_requires, + 'python_requires': '>=3.6,<4.0', +} + + +setup(**setup_kwargs) \ No newline at end of file diff --git a/tools/c7n_openstack/test/data.flight/default.yml b/tools/c7n_openstack/test/data.flight/default.yml new file mode 100644 index 00000000000..6c890a1aeec --- /dev/null +++ b/tools/c7n_openstack/test/data.flight/default.yml @@ -0,0 +1,336 @@ +interactions: + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:5000/ + response: + body: {string: '{"versions": {"values": [{"id": "v3.14", "status": "stable", "updated": "2020-04-07T00:00:00Z", "links": [{"rel": "self", "href": "http://127.0.0.1:5000/v3/"}], "media-types": [{"base": "application/json", "type": "application/vnd.openstack.identity-v3+j +son"}]}]}}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: POST + uri: http://keystone:5000/v3/auth/tokens + response: + body: {string: '{"token": {"methods": ["password"], "user": {"domain": {"id": "default", "name": "Default"}, "id": "0ca05c20ee48419bb171707560ad793b", "name": "admin", "password_expires_at": null}, "audit_ids": ["EisDtEpHTcu-_VxIz4jP8w"], "expires_at": "2020-11-26T04:43:09.000000Z", "issued_at": "2020-11-26T03:43:09.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "3d1d9e8cf44143abbd582e026fa507a3", "name": "admin"}, "is_domain": false, "roles": [{"id": "6135c43502b64aafb105bab98efd8595", "name": "admin"}, {"id": "13df80fdc9064e0482e1485a6294adfd", "name": "reader"}, {"id": "0988d05f27434167b372588bff13f967", "name": "member"}], "catalog": [{"endpoints": [{"id": "0c167064c60b4d31adaac7b9f9e695a4", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8080/v1/AUTH_3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "a72dda2f782e4350a41aad1d6d85ce5a", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8080/v1/AUTH_3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "b18e9aff73af4b57b456b58b800e99bf", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8080/v1/AUTH_3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}], "id": "31375fead0d346a68ee168ce9ef6ff48", "type": "object-store", "name": "swift"}, {"endpoints": [{"id": "042b6cf9048c4f63baf576de2d69cd48", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8042", "region": "RegionOne"}, {"id": "396ae5a7a9cc48fe8baca3d0f2216fe4", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8042", "region": "RegionOne"}, {"id": "4824a2fd92ca418dbdf7c3660dbe1d21", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8042", "region": "RegionOne"}], "id": "3f40fed5c6274bb9a59dea54628412f5", "type": "alarming", "name": "aodh"}, {"endpoints": [{"id": "119020311c48489f87025fc48eaf581b", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "2b2c5d0336f340649c73aae455336ac0", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "7a54f1f96fb2448c8c6311376fd2ec79", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}], "id": "4b27a00dfcf94873a722fbf2b2998642", "type": "compute", "name": "nova"}, {"endpoints": [{"id": "4003b4daea5c4f70af92fd6400d33ace", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:5000", "region": "RegionOne"}, {"id": "938c74331a1b47e8b7aa78ad7c9ee732", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:5000", "region": "RegionOne"}, {"id": "9ed2d867c48649c4bc6d255d587c2f52", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:5000", "region": "RegionOne"}], "id": "58b849d76f45497e9db8bba2d300e344", "type": "identity", "name": "keystone"}, {"endpoints": [{"id": "61a72ea2fba54dc1a45803ac3277fc9a", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8777", "region": "RegionOne"}, {"id": "f86dcc5035fb4473b1850ebd6fcd221c", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8777", "region": "RegionOne"}, {"id": "f8e194657e354f52a98808352d81a0a0", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8777", "region": "RegionOne"}], "id": "6b48671f1315400ea4cf177d13b6e186", "type": "metering", "name": "ceilometer"}, {"endpoints": [{"id": "6095ce8254bb45b1b8aed529696b0ab3", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8776/v3/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "671156be632b477fa4f075a919375bc2", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8776/v3/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "85f9742e2e004fbc82a93af08c104dc6", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8776/v3/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}], "id": "6c534f6b3152422fad61ceaacdfd2d8f", "type": "volumev3", "name": "cinderv3"}, {"endpoints": [{"id": "71a1a6b39c9e4e5a8124b4b5094fd053", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8041", "region": "RegionOne"}, {"id": "cc214c3b987848d08d79befabe948863", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8041", "region": "RegionOne"}, {"id": "d4edaf8639c84489b7a6d6ea6008b0ae", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8041", "region": "RegionOne"}], "id": "831cb70d2d244cada780895245c0847c", "type": "metric", "name": "gnocchi"}, {"endpoints": [{"id": "1380f800fe5a4c6785c3f0abbd12e65d", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:9292", "region": "RegionOne"}, {"id": "332f3119d5da4291960abd615faca249", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:9292", "region": "RegionOne"}, {"id": "a49b45320fc543ae9d890ab46e60c481", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:9292", "region": "RegionOne"}], "id": "aae882bc60a84408859f0012ff78c584", "type": "image", "name": "glance"}, {"endpoints": [{"id": "4b5088db5d564c9c8c5f3eac11e4aa5d", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8776/v2/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "63f25449f6764500bf5a7213f23083ec", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8776/v2/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}, {"id": "ac191ab78c6a4d30afe43b3635f7a704", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8776/v2/3d1d9e8cf44143abbd582e026fa507a3", "region": "RegionOne"}], "id": "b764ed8b7aa4422b858d8d0624d1040f", "type": "volumev2", "name": "cinderv2"}, {"endpoints": [{"id": "0709743b3a4c4b22884f7d811edb3614", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:9696", "region": "RegionOne"}, {"id": "8d5316efc34b460381e9e61975af3418", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:9696", "region": "RegionOne"}, {"id": "ca381ed0a9b44224b5cc931e0098f88c", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:9696", "region": "RegionOne"}], "id": "c9880ff58de14965ab9acd8e3d5dc73e", "type": "network", "name": "neutron"}, {"endpoints": [{"id": "20d58a19e9204b0b983fefa5c2bb02bb", "interface": "admin", "region_id": "RegionOne", "url": "http://keystone:8778/placement", "region": "RegionOne"}, {"id": "aa5f7d1ad8bd489aae40750479ebb46c", "interface": "public", "region_id": "RegionOne", "url": "http://keystone:8778/placement", "region": "RegionOne"}, {"id": "ed99a782cdef4cda908b7e55156862d2", "interface": "internal", "region_id": "RegionOne", "url": "http://keystone:8778/placement", "region": "RegionOne"}], "id": "cea3b185e75e4cb4a9e8f6d6feded397", "type": "placement", "name": "placement"}]}}'} + headers: + X-Subject-Token: "test-token" + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/servers/detail + response: + body: {string: '{"servers": [{"id": "6adf8322-e7d4-40e4-8f60-66b22eca6798", "name": "c7n-test-1", "status": "ACTIVE", "tenant_id": "3d1d9e8cf44143abbd582e026fa507a3", "user_id": "0ca05c20ee48419bb171707560ad793b", "metadata": {"name": "name", "en": "123", "name1": "test033", "name2": "test033", "name3": "test033", "name4": "test034", "name5": "test035", "name6": "test036"}, "hostId": "665cf485a8bf692d8f0feec5fbeba68343775ea6d40362c39a534b0f", "image": {"id": "d436429c-905a-4fe2-8c76-d7bcd677c879", "links": [{"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/images/d436429c-905a-4fe2-8c76-d7bcd677c879"}]}, "flavor": {"vcpus": 1, "ram": 512, "disk": 1, "ephemeral": 0, "swap": 0, "original_name": "m1.tiny", "extra_specs": {}}, "created": "2020-11-19T12:18:10Z", "updated": "2020-11-26T03:41:58Z", "addresses": {"MSKJ_PROD_NETWORK": [{"version": 4, "addr": "192.168.100.100", "OS-EXT-IPS:type": "fixed", "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:cb:e3:8d"}]}, "accessIPv4": "", "accessIPv6": "", "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/servers/6adf8322-e7d4-40e4-8f60-66b22eca6798"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/servers/6adf8322-e7d4-40e4-8f60-66b22eca6798"}], "OS-DCF:diskConfig": "MANUAL", "progress": 0, "OS-EXT-AZ:availability_zone": "nova", "config_drive": "", "key_name": null, "OS-SRV-USG:launched_at": "2020-11-19T12:18:19.000000", "OS-SRV-USG:terminated_at": null, "OS-EXT-SRV-ATTR:host": "ip-192-168-193-40.cn-northwest-1.compute.internal", "OS-EXT-SRV-ATTR:instance_name": "instance-0000001a", "OS-EXT-SRV-ATTR:hypervisor_hostname": "ip-192-168-193-40.cn-northwest-1.compute.internal", "OS-EXT-SRV-ATTR:reservation_id": "r-3feknjtl", "OS-EXT-SRV-ATTR:launch_index": 0, "OS-EXT-SRV-ATTR:hostname": "c7n-test-1", "OS-EXT-SRV-ATTR:kernel_id": "", "OS-EXT-SRV-ATTR:ramdisk_id": "", "OS-EXT-SRV-ATTR:root_device_name": "/dev/vda", "OS-EXT-SRV-ATTR:user_data": "", "OS-EXT-STS:task_state": null, "OS-EXT-STS:vm_state": "active", "OS-EXT-STS:power_state": 0, "os-extended-volumes:volumes_attached": [], "locked": false, "locked_reason": null, "description": "c7n-test-1", "tags": ["name0=3", "name13333333=35", "name1=33", "name1=35", "name2=2", "name2=333333333"], "trusted_image_certificates": null, "host_status": "UP", "security_groups": [{"name": "MSKJ_BASE_SG"}]}, {"id": "4533efe6-e108-4552-857a-26fe297ca45e", "name": "c7n-test-2", "status": "ACTIVE", "tenant_id": "3d1d9e8cf44143abbd582e026fa507a3", "user_id": "0ca05c20ee48419bb171707560ad793b", "metadata": {}, "hostId": "665cf485a8bf692d8f0feec5fbeba68343775ea6d40362c39a534b0f", "image": {"id": "415f7db5-2d3b-411e-a5f5-80321dbfec0b", "links": [{"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/images/415f7db5-2d3b-411e-a5f5-80321dbfec0b"}]}, "flavor": {"vcpus": 1, "ram": 1024, "disk": 20, "ephemeral": 0, "swap": 0, "original_name": "m1.micro", "extra_specs": {}}, "created": "2020-11-05T03:48:57Z", "updated": "2020-11-24T07:19:45Z", "addresses": {"external_network": [{"version": 4, "addr": "192.168.193.77", "OS-EXT-IPS:type": "fixed", "OS-EXT-IPS-MAC:mac_addr": "02:97:2b:98:2e:5c"}]}, "accessIPv4": "", "accessIPv6": "", "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/servers/4533efe6-e108-4552-857a-26fe297ca45e"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/servers/4533efe6-e108-4552-857a-26fe297ca45e"}], "OS-DCF:diskConfig": "MANUAL", "progress": 0, "OS-EXT-AZ:availability_zone": "nova", "config_drive": "", "key_name": "default", "OS-SRV-USG:launched_at": "2020-11-05T03:49:23.000000", "OS-SRV-USG:terminated_at": null, "OS-EXT-SRV-ATTR:host": "ip-192-168-193-40.cn-northwest-1.compute.internal", "OS-EXT-SRV-ATTR:instance_name": "instance-00000018", "OS-EXT-SRV-ATTR:hypervisor_hostname": "ip-192-168-193-40.cn-northwest-1.compute.internal", "OS-EXT-SRV-ATTR:reservation_id": "r-rh305at8", "OS-EXT-SRV-ATTR:launch_index": 0, "OS-EXT-SRV-ATTR:hostname": "int32bit-test-1", "OS-EXT-SRV-ATTR:kernel_id": "", "OS-EXT-SRV-ATTR:ramdisk_id": "", "OS-EXT-SRV-ATTR:root_device_name": "/dev/vda", "OS-EXT-SRV-ATTR:user_data": null, "OS-EXT-STS:task_state": null, "OS-EXT-STS:vm_state": "active", "OS-EXT-STS:power_state": 1, "os-extended-volumes:volumes_attached": [{"id": "2ef1004e-013b-4b77-9e54-b115869a6504", "delete_on_termination": false}], "locked": false, "locked_reason": null, "description": null, "tags": ["a=a", "b=b", "c=c"], "trusted_image_certificates": null, "host_status": "UP", "security_groups": [{"name": "default"}]}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/ports.json?device_id=6adf8322-e7d4-40e4-8f60-66b22eca6798 + response: + body: {string: ' {"ports":[{"id":"2940edf7-7dda-4dee-a05e-9c385c4c0b6b","name":"port1","network_id":"2590b53d-2282-4fa5-9251-2b9fbd8e3cec","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","mac_address":"fa:16:3e:cb:e3:8d","admin_state_up":true,"status":"DOWN","device_id":"6adf8322-e7d4-40e4-8f60-66b22eca6798","device_owner":"compute:nova","fixed_ips":[{"subnet_id":"267a0c4b-fcdd-4b33-9ac6-409cbd8cda3c","ip_address":"192.168.100.100"}],"allowed_address_pairs":[],"extra_dhcp_opts":[],"security_groups":["74e09d96-1ae7-497d-a6d2-ac212277de1b"],"description":"","binding:vnic_type":"normal","binding:profile":{},"binding:host_id":"ip-192-168-193-40.cn-northwest-1.compute.internal","binding:vif_type":"ovs","binding:vif_details":{"port_filter":true},"port_security_enabled":true,"qos_policy_id":null,"qos_network_policy_id":null,"resource_request":null,"tags":["123","blue"],"created_at":"2020-11-19T12:17:57Z","updated_at":"2020-11-26T02:40:10Z","revision_number":33,"project_id":"3d1d9e8cf44143abbd582e026fa507a3"}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/ports.json?device_id=4533efe6-e108-4552-857a-26fe297ca45e + response: + body: {string: '{"ports":[{"id":"695c8e70-ae94-4c39-86f0-c15dcaea7dd2","name":"","network_id":"1addba70-cc5e-40aa-a630-6806d68201c6","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","mac_address":"02:97:2b:98:2e:5c","admin_state_up":true,"status":"ACTIVE","device_id":"4533efe6-e108-4552-857a-26fe297ca45e","device_owner":"compute:nova","fixed_ips":[{"subnet_id":"ee781473-a79c-4a40-a33d-01ae100f0d79","ip_address":"192.168.193.77"}],"allowed_address_pairs":[],"extra_dhcp_opts":[],"security_groups":["bd4cfd7d-be5d-48f0-a85e-48e212c5dde2"],"description":"","binding:vnic_type":"normal","binding:profile":{},"binding:host_id":"ip-192-168-193-40.cn-northwest-1.compute.internal","binding:vif_type":"ovs","binding:vif_details":{"port_filter":true},"port_security_enabled":true,"qos_policy_id":null,"qos_network_policy_id":null,"resource_request":null,"tags":[],"created_at":"2020-11-04T11:54:11Z","updated_at":"2020-11-19T01:04:09Z","revision_number":31,"project_id":"3d1d9e8cf44143abbd582e026fa507a3"}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/networks.json + response: + body: {string: '{"networks":[{"id":"1addba70-cc5e-40aa-a630-6806d68201c6","name":"external_network","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","admin_state_up":true,"mtu":1500,"status":"ACTIVE","subnets":["ee781473-a79c-4a40-a33d-01ae100f0d79"],"shared":false,"availability_zone_hints":[],"availability_zones":[],"ipv4_address_scope":null,"ipv6_address_scope":null,"router:external":true,"description":"","port_security_enabled":true,"qos_policy_id":null,"is_default":false,"tags":[],"created_at":"2020-11-04T10:45:01Z","updated_at":"2020-11-05T03:36:23Z","revision_number":5,"project_id":"3d1d9e8cf44143abbd582e026fa507a3","provider:network_type":"flat","provider:physical_network":"extnet","provider:segmentation_id":null},{"id":"2590b53d-2282-4fa5-9251-2b9fbd8e3cec","name":"MSKJ_PROD_NETWORK","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","admin_state_up":true,"mtu":1442,"status":"ACTIVE","subnets":["267a0c4b-fcdd-4b33-9ac6-409cbd8cda3c"],"shared":false,"availability_zone_hints":[],"availability_zones":[],"ipv4_address_scope":null,"ipv6_address_scope":null,"router:external":false,"description":"","port_security_enabled":true,"qos_policy_id":null,"tags":["1122","11=2","1122k"],"created_at":"2020-11-19T12:12:28Z","updated_at":"2020-11-26T06:54:09Z","revision_number":5,"project_id":"3d1d9e8cf44143abbd582e026fa507a3","provider:network_type":"geneve","provider:physical_network":null,"provider:segmentation_id":2000}]}' } + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/floatingips.json?port_id=2940edf7-7dda-4dee-a05e-9c385c4c0b6b + response: + body: {string: '{"floatingips": []}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/floatingips.json?port_id=695c8e70-ae94-4c39-86f0-c15dcaea7dd2 + response: + body: {string: '{"floatingips": []}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:9696/v2.0/subnets.json + response: + body: {string: '{"subnets":[{"id":"267a0c4b-fcdd-4b33-9ac6-409cbd8cda3c","name":"MSKJ_PROD_PRIVATE_SUBNET_1","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","network_id":"2590b53d-2282-4fa5-9251-2b9fbd8e3cec","ip_version":4,"subnetpool_id":null,"enable_dhcp":true,"ipv6_ra_mode":null,"ipv6_address_mode":null,"gateway_ip":"192.168.100.1","cidr":"192.168.100.0/24","allocation_pools":[{"start":"192.168.100.2","end":"192.168.100.254"}],"host_routes":[],"dns_nameservers":[],"description":"","service_types":[],"tags":[],"created_at":"2020-11-19T12:12:35Z","updated_at":"2020-11-19T12:12:35Z","revision_number":0,"project_id":"3d1d9e8cf44143abbd582e026fa507a3"},{"id":"ee781473-a79c-4a40-a33d-01ae100f0d79","name":"public_subnet","tenant_id":"3d1d9e8cf44143abbd582e026fa507a3","network_id":"1addba70-cc5e-40aa-a630-6806d68201c6","ip_version":4,"subnetpool_id":null,"enable_dhcp":true,"ipv6_ra_mode":null,"ipv6_address_mode":null,"gateway_ip":"192.168.193.1","cidr":"192.168.193.0/25","allocation_pools":[{"start":"192.168.193.77","end":"192.168.193.79"}],"host_routes":[],"dns_nameservers":[],"description":"","service_types":[],"tags":[],"created_at":"2020-11-04T10:46:10Z","updated_at":"2020-11-05T03:36:23Z","revision_number":3,"project_id":"3d1d9e8cf44143abbd582e026fa507a3"}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/detail?is_public=None + response: + body: {string: '{"flavors": [{"id": "1", "name": "m1.tiny", "ram": 512, "disk": 1, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 1, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/1"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/1"}], "description": "tt", "extra_specs": {}}, {"id": "2", "name": "m1.small", "ram": 2048, "disk": 20, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 1, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/2"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/2"}], "description": null, "extra_specs": {}}, {"id": "3", "name": "m1.medium", "ram": 4096, "disk": 40, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 2, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/3"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/3"}], "description": null, "extra_specs": {}}, {"id": "4", "name": "m1.large", "ram": 8192, "disk": 80, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 4, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/4"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/4"}], "description": null, "extra_specs": {}}, {"id": "5", "name": "m1.xlarge", "ram": 16384, "disk": 160, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 8, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/5"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/5"}], "description": null, "extra_specs": {}}, {"id": "2", "name": "m1.small", "ram": 1024, "disk": 20, "swap": 0, "OS-FLV-EXT-DATA:ephemeral": 0, "OS-FLV-DISABLED:disabled": false, "vcpus": 1, "os-flavor-access:is_public": true, "rxtx_factor": 1.0, "links": [{"rel": "self", "href": "http://192.168.193.40:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/2"}, {"rel": "bookmark", "href": "http://192.168.193.40:8774/3d1d9e8cf44143abbd582e026fa507a3/flavors/2"}], "description": null, "extra_specs": {}}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/1/os-extra_specs + response: + body: {string: '{"extra_specs":[]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/2/os-extra_specs + response: + body: {string: '{"extra_specs":[]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/3/os-extra_specs + response: + body: {string: '{"extra_specs":[]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/5/os-extra_specs + response: + body: {string: '{"extra_specs":[]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8774/v2.1/3d1d9e8cf44143abbd582e026fa507a3/flavors/4/os-extra_specs + response: + body: {string: '{"extra_specs":[]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:5000/v3/projects + response: + body: {string: '{"projects": [{"id": "3d1d9e8cf44143abbd582e026fa507a3", "name": "admin", "domain_id": "default", "description": "Bootstrap project for initializing the cloud.", "enabled": true, "parent_id": "default", "is_domain": false, "tags": [], "options": {}, "links": {"self": "http://127.0.0.1:5000/v3/projects/3d1d9e8cf44143abbd582e026fa507a3"}}, {"id": "57626756800343fd8474e0654f67c89a", "name": "demo", "domain_id": "default", "description": "default tenant", "enabled": true, "parent_id": "default", "is_domain": false, "tags": [], "options": {}, "links": {"self": "http://127.0.0.1:5000/v3/projects/57626756800343fd8474e0654f67c89a"}}, {"id": "cdf7c8b6db234a3695420f45b42e603b", "name": "services", "domain_id": "default", "description": "", "enabled": true, "parent_id": "default", "is_domain": false, "tags": [], "options": {}, "links": {"self": "http://127.0.0.1:5000/v3/projects/cdf7c8b6db234a3695420f45b42e603b"}}], "links": {"next": null, "self": "http://127.0.0.1:5000/v3/projects", "previous": null}}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: GET + uri: http://keystone:8776/v3/3d1d9e8cf44143abbd582e026fa507a3/volumes/detail + response: + body: {string: '{"volumes": [{"id": "7f64ece8-e8bf-4d0e-b358-8028e9769b42", "status": "available", "size": 1, "availability_zone": "nova", "created_at": "2020-11-27T04:26:03.000000", "updated_at": "2020-11-27T04:26:04.000000", "attachments": [], "name": "c7n-test-1", "description": null, "volume_type": "iscsi", "snapshot_id": null, "source_volid": null, "metadata": {}, "links": [{"rel": "self", "href": "http://192.168.193.40:8776/v3/3d1d9e8cf44143abbd582e026fa507a3/volumes/7f64ece8-e8bf-4d0e-b358-8028e9769b42"}, {"rel": "bookmark", "href": "http://192.168.193.40:8776/3d1d9e8cf44143abbd582e026fa507a3/volumes/7f64ece8-e8bf-4d0e-b358-8028e9769b42"}], "user_id": "0ca05c20ee48419bb171707560ad793b", "bootable": "false", "encrypted": false, "replication_status": null, "consistencygroup_id": null, "multiattach": false, "migration_status": null, "group_id": null, "provider_id": null, "shared_targets": false, "service_uuid": "e3e266a0-510e-43bd-9386-d292672909ca", "os-vol-tenant-attr:tenant_id": "3d1d9e8cf44143abbd582e026fa507a3", "os-vol-mig-status-attr:migstat": null, "os-vol-mig-status-attr:name_id": null, "os-vol-host-attr:host": "ip-192-168-193-40.cn-northwest-1.compute.internal@lvm#lvm"}, {"id": "2ef1004e-013b-4b77-9e54-b115869a6504", "status": "in-use", "size": 1, "availability_zone": "nova", "created_at": "2020-11-24T13:54:37.000000", "updated_at": "2020-11-24T14:01:59.000000", "attachments": [{"id": "2ef1004e-013b-4b77-9e54-b115869a6504", "attachment_id": "56cd49b2-9e50-4084-b08c-060247c7af81", "volume_id": "2ef1004e-013b-4b77-9e54-b115869a6504", "server_id": "4533efe6-e108-4552-857a-26fe297ca45e", "host_name": "ip-192-168-193-40.cn-northwest-1.compute.internal", "device": "/dev/vdb", "attached_at": "2020-11-24T14:01:58.000000"}], "name": "gaoyan-test-volume01", "description": null, "volume_type": "iscsi", "snapshot_id": null, "source_volid": null, "metadata": {}, "links": [{"rel": "self", "href": "http://192.168.193.40:8776/v3/3d1d9e8cf44143abbd582e026fa507a3/volumes/2ef1004e-013b-4b77-9e54-b115869a6504"}, {"rel": "bookmark", "href": "http://192.168.193.40:8776/3d1d9e8cf44143abbd582e026fa507a3/volumes/2ef1004e-013b-4b77-9e54-b115869a6504"}], "user_id": "0ca05c20ee48419bb171707560ad793b", "bootable": "false", "encrypted": false, "replication_status": null, "consistencygroup_id": null, "multiattach": false, "migration_status": null, "group_id": null, "provider_id": null, "shared_targets": false, "service_uuid": "e3e266a0-510e-43bd-9386-d292672909ca", "os-vol-tenant-attr:tenant_id": "3d1d9e8cf44143abbd582e026fa507a3", "os-vol-mig-status-attr:migstat": null, "os-vol-mig-status-attr:name_id": null, "os-vol-host-attr:host": "ip-192-168-193-40.cn-northwest-1.compute.internal@lvm#lvm"}]}'} + headers: + Content-Type: [application/json] + status: {code: 200, message: OK} + - request: + body: null + headers: + Accept: [application/json] + Content-Type: [application/json] + method: DELETE + uri: http://keystone:8776/v3/3d1d9e8cf44143abbd582e026fa507a3/volumes/7f64ece8-e8bf-4d0e-b358-8028e9769b42 + response: + body: {string: ''} + headers: + Content-Type: [test/html] + status: {code: 200, message: OK} +version: 1 + 33 tools/c7n_openstack/test/test_project.py + Viewed + @@ -0,0 +1,33 @@ + # Copyright The Cloud Custodian Authors. + # SPDX-License-Identifier: Apache-2.0 + from common_openstack import OpenStackTest + + +class ProjectTest(OpenStackTest): + + def test_project_query(self): + factory = self.replay_flight_data() + p = self.load_policy({ + 'name': 'all-projects', + 'resource': 'openstack.project'}, + session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 3) + + def test_project_filter_by_name(self): + factory = self.replay_flight_data() + policy = { + 'name': 'project-demo', + 'resource': 'openstack.project', + 'filters': [ + { + "type": "value", + "key": "name", + "value": "demo", + }, + ], + } + p = self.load_policy(policy, session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 1) + self.assertEqual(resources[0].name, "demo") + 77 tools/c7n_openstack/test/test_server.py + Viewed + @@ -0,0 +1,77 @@ + # Copyright The Cloud Custodian Authors. + # SPDX-License-Identifier: Apache-2.0 + from common_openstack import OpenStackTest + + +class ServerTest(OpenStackTest): + + def test_server_query(self): + factory = self.replay_flight_data() + p = self.load_policy({ + 'name': 'all-servers', + 'resource': 'openstack.server'}, + session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 2) + + def test_server_filter_name(self): + factory = self.replay_flight_data() + policy = { + 'name': 'get-server-c7n-test-1', + 'resource': 'openstack.server', + 'filters': [ + { + "type": "value", + "key": "name", + "value": "c7n-test-1", + }, + ], + } + p = self.load_policy(policy, session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 1) + self.assertEqual(resources[0].name, "c7n-test-1") + + def test_server_filter_flavor(self): + factory = self.replay_flight_data() + policy = { + 'name': 'get-server-c7n-test-1', + 'resource': 'openstack.server', + 'filters': [ + { + "type": "flavor", + "flavor_name": "m1.tiny", + }, + ], + } + p = self.load_policy(policy, session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 1) + self.assertEqual(resources[0].name, "c7n-test-1") + + def test_server_filter_tags(self): + factory = self.replay_flight_data() + policy = { + 'name': 'get-server-c7n-test-1', + 'resource': 'openstack.server', + 'filters': [ + { + "type": "tags", + "tags": [ + { + "key": "a", + "value": "a", + }, + { + "key": "b", + "value": "b", + }, + ], + "op": "all", + }, + ], + } + p = self.load_policy(policy, session_factory=factory) + resources = p.run() + self.assertEqual(len(resources), 1) + self.assertEqual(resources[0].name, "c7n-test-2") \ No newline at end of file diff --git a/tools/c7n_openstack/test/openstack.py b/tools/c7n_openstack/test/openstack.py new file mode 100644 index 00000000000..c9013d2720d --- /dev/null +++ b/tools/c7n_openstack/test/openstack.py @@ -0,0 +1,66 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +import logging +import os + +import openstack + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S') +# 本地测试用例 +def _loadFile_(): + json = dict() + f = open("/opt/fit2cloud/openstack.txt") + lines = f.readlines() + for line in lines: + line = line.strip() + if "openstack.OS_USERNAME" in line: + OS_USERNAME = line[line.rfind('=') + 1:] + json['OS_USERNAME'] = OS_USERNAME + if "openstack.OS_PASSWORD" in line: + OS_PASSWORD = line[line.rfind('=') + 1:] + json['OS_PASSWORD'] = OS_PASSWORD + if "openstack.OS_REGION_NAME" in line: + OS_REGION_NAME = line[line.rfind('=') + 1:] + json['OS_REGION_NAME'] = OS_REGION_NAME + if "openstack.OS_AUTH_URL" in line: + OS_AUTH_URL = line[line.rfind('=') + 1:] + json['OS_AUTH_URL'] = OS_AUTH_URL + if "openstack.OS_PROJECT_NAME" in line: + OS_PROJECT_NAME = line[line.rfind('=') + 1:] + json['OS_PROJECT_NAME'] = OS_PROJECT_NAME + f.close() + print('认证信息: ' + str(json)) + return json + +params = _loadFile_() + +OPENSTACK_CONFIG = { + 'OS_USERNAME': params['OS_USERNAME'], + 'OS_PASSWORD': params['OS_PASSWORD'], + 'OS_REGION_NAME': params['OS_REGION_NAME'], + 'OS_AUTH_URL': params['OS_AUTH_URL'], #http://keystone:5000/v3 + 'OS_PROJECT_NAME': params['OS_PROJECT_NAME'], + 'OS_USER_DOMAIN_NAME': 'Default', + 'OS_PROJECT_DOMAIN_NAME': 'Default', + 'OS_IDENTITY_API_VERSION': '3', + 'OS_CLOUD_NAME': 'c7n-cloud', +} + +DEFAULT_CASSETTE_FILE = "default.yaml" + +def init_openstack_config(): + for k, v in OPENSTACK_CONFIG.items(): + os.environ[k] = v + +def client(): + cloud = openstack.connect(cloud=os.getenv('OS_CLOUD_NAME')) + return cloud + +def list_users(self=None): + print("List Users:") + for user in client().list_users(self): + print(user) + +if __name__ == '__main__': + logging.info("Hello OpenStack OpenApi!") + list_users(None) \ No newline at end of file diff --git a/tools/c7n_tencent/.gitignore b/tools/c7n_tencent/.gitignore index 1e7bb232c6a..fb3bc159d15 100644 --- a/tools/c7n_tencent/.gitignore +++ b/tools/c7n_tencent/.gitignore @@ -2,3 +2,4 @@ *py~ __pycache__ *.egg-info +tencent.py \ No newline at end of file diff --git a/tools/c7n_tencent/__init__.py b/tools/c7n_tencent/__init__.py new file mode 100644 index 00000000000..fbe549677de --- /dev/null +++ b/tools/c7n_tencent/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2017-2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file diff --git a/tools/c7n_tencent/c7n_tencent/client.py b/tools/c7n_tencent/c7n_tencent/client.py index 3c43cdf5234..f84283118e9 100644 --- a/tools/c7n_tencent/c7n_tencent/client.py +++ b/tools/c7n_tencent/c7n_tencent/client.py @@ -25,8 +25,10 @@ # 导入可选配置类 # 导入对应产品模块的 client models。 from tencentcloud.cvm.v20170312 import cvm_client +from tencentcloud.es.v20180416 import es_client from tencentcloud.mongodb.v20190725 import mongodb_client from tencentcloud.monitor.v20180724 import monitor_client +from tencentcloud.postgres.v20170312 import postgres_client from tencentcloud.redis.v20180412 import redis_client from tencentcloud.vpc.v20170312 import vpc_client @@ -75,14 +77,17 @@ def client(self, service): client = mongodb_client.MongodbClient(cred, os.getenv('TENCENT_DEFAULT_REGION')) elif 'redis_client' in service: client = redis_client.RedisClient(cred, os.getenv('TENCENT_DEFAULT_REGION')) + elif 'postgres_client' in service: + client = postgres_client.PostgresClient(cred, os.getenv("TENCENT_DEFAULT_REGION")) + elif 'es_client' in service: + client = es_client.EsClient(cred, os.getenv('TENCENT_DEFAULT_REGION')) elif 'coss3_client' in service: # 1. 设置用户配置, 包括 secretId，secretKey 以及 Region endpoint = 'cos.' + os.getenv('TENCENT_DEFAULT_REGION') + '.myqcloud.com' config = CosConfig(Region=os.getenv('TENCENT_DEFAULT_REGION'), SecretId=os.getenv('TENCENT_SECRETID'), - SecretKey=os.getenv('TENCENT_SECRETKEY'),Endpoint=endpoint) + SecretKey=os.getenv('TENCENT_SECRETKEY'), Endpoint=endpoint) # 2. 获取客户端对象 client = CosS3Client(config) else: client = cvm_client.CvmClient(cred, os.getenv('TENCENT_DEFAULT_REGION')) return client - diff --git a/tools/c7n_tencent/c7n_tencent/filter_util.py b/tools/c7n_tencent/c7n_tencent/filter_util.py new file mode 100644 index 00000000000..f3413e1a489 --- /dev/null +++ b/tools/c7n_tencent/c7n_tencent/filter_util.py @@ -0,0 +1,196 @@ +import jsonpath +import json +import time + + +def get_value(res_i, value_key): + """ + 获取数据 + @param res_i: + @param value_key: + @return: + """ + json_str = json.dumps(res_i) + u_str = json.loads(json_str) + if str(value_key).startswith("$."): + return jsonpath.jsonpath(u_str, value_key) + return jsonpath.jsonpath(u_str, '$.' + value_key) + + +def like(filter_obj, i): + """ + 判断是否匹配 + @param filter_obj: + @param i: + @return: + """ + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + if cloud_value: + if len(cloud_value) == 0: + return False + like_value = filter_obj.data.get('like', '') + # 如果不传入数据默认不过滤 + if like_value: + return like_value in cloud_value[0] + return True + + +def eq(filter_obj, i): + """ + 判断是否相等 + @param filter_obj: + @param i: + @return: + """ + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + if cloud_value: + if len(cloud_value) == 0: + return False + value = filter_obj.data.get('value', '') + # 如果不传入数据默认不过滤 + if value: + return value == cloud_value[0] + return True + + +def time_to_num(time_str, format): + # 2017-12-04 00:00:00 + print(time_str) + timeArray = time.strptime(time_str, format) + return int(time.mktime(timeArray)) + + +def region(filter_obj, i): + """ + 判断是否是在某个范围 + @param filter_obj: + @param i: + @return: + """ + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + ge = filter_obj.data.get('ge', '') + le = filter_obj.data.get('le', '') + if len(cloud_value) == 0: + return False + cloud_value = cloud_value[0] + if cloud_value and len(str(cloud_value)) > 0 and isinstance(cloud_value, str): + cloud_value = time_to_num(cloud_value, '%Y-%m-%d %H:%M:%S') + # 如果不传入数据默认不过滤 + if cloud_value and (isinstance(cloud_value, int) or isinstance(cloud_value, float)): + if ge and le: + return ge <= cloud_value <= le + elif ge: + return ge <= cloud_value + elif le: + return cloud_value <= le + return True + + +def is_null(filter_obj, i): + """ + 判断变量是否存在 + @param filter_obj: + @param i: + @return: + """ + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + if cloud_value: + is_empty = filter_obj.data.get('is_empty', '') + if is_empty and len(cloud_value) == 0: + return i + elif not is_empty and len(cloud_value) != 0: + return i + else: + return False + + +def in_like(filter_obj, i): + in_like_value = filter_obj.data.get('in_like', '') + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + if cloud_value: + if isinstance(cloud_value[0], list): + for index in range(list(cloud_value[0])): + if in_like_value in cloud_value[0][index]: + return i + else: + False + + +def in_eq(filter_obj, i): + in_value = filter_obj.data.get('in', '') + cloud_value = get_value(i, filter_obj.schema['properties']['type']['enum'][0]) + if cloud_value: + if isinstance(cloud_value[0], list): + if list(cloud_value[0]).__contains__(in_value): + return i + else: + False + + +def filter_res(filter_obj, i): + """ + 过滤返回对象 + @param filter_obj: + @param i: + @return: + """ + like_value = filter_obj.data.get('like', '') + value = filter_obj.data.get('value', '') + ge = filter_obj.data.get('ge', '') + le = filter_obj.data.get('le', '') + is_empty = filter_obj.data.get('is_empty', '') + in_like_field = filter_obj.data.get('in_like', '') + in_eq_field = filter_obj.data.get('in', '') + if like_value: + if like(filter_obj, i): + return i + else: + return False + elif value: + if eq(filter_obj, i): + return i + else: + return False + elif ge or le: + if region(filter_obj, i): + return i + else: + return False + elif is_empty: + if is_null(filter_obj, i): + return i + else: + return False + elif in_like_field: + if in_like(filter_obj, i): + return i + else: + return False + elif in_eq_field: + if in_eq(filter_obj, i): + return i + else: + return False + + return i + + +types = { + 'is_empty': {'is_empty': {'type': 'boolean'}}, + 'boolean': {'value': {'type': 'boolean'}, 'is_empty': {'type': 'boolean'}}, + 'number': {'ge': {'type': 'number'}, 'le': {'type': 'number'}, 'is_empty': {'type': 'boolean'}}, + 'string': {'like': {'type': 'string'}, 'value': {'type': 'string'}, 'is_empty': {'type': 'boolean'}}, + 'list_string': {'in': {'type': 'string'}, 'in_like': {'type': {'type': 'string'}}, + 'is_empty': {'type': 'boolean'}}, + 'list_number': {'in': {'type': 'number'}, 'in_like': {'type': 'number'}, 'is_empty': {'type': 'boolean'}}, + 'time': {'ge': {'type': 'number'}, 'le': {'type': 'number'}, 'is_empty': {'type': 'boolean'}} +} + + +def get_schema(type): + """ + 获取类型 + @param type: + @return: + """ + return types.get(type) diff --git a/tools/c7n_tencent/c7n_tencent/filters/filter.py b/tools/c7n_tencent/c7n_tencent/filters/filter.py index f34fc390917..d684734d456 100644 --- a/tools/c7n_tencent/c7n_tencent/filters/filter.py +++ b/tools/c7n_tencent/c7n_tencent/filters/filter.py @@ -1,9 +1,11 @@ import datetime import json +import re from concurrent.futures import as_completed from datetime import timedelta import jmespath +from c7n_tencent import filter_util from dateutil.parser import parse from dateutil.tz import tzutc @@ -23,6 +25,36 @@ def validate(self): def __call__(self, i): return self.get_request(i) + +class TencentEsFilter(Filter): + schema = None + + def get_request(self, i): + return filter_util.filter_res(self, i) + + def validate(self): + keys = ['is_empty', 'value', 'like', 'ge', 'le', 'in', 'in_like'] + hits = [] + for key in self.data: + if keys == 'type': + continue + if keys.__contains__(key): + hits.append(key) + # ge 和 le是可以同时出现的 + if len(hits) > 1: + if len(hits) == 2: + if not list(hits).__contains__('ge') or not list(hits).__contains__('le'): + raise PolicyValidationError( + '过滤类型只能配置一个,ge 和 le 可以同时使用,', self.data) + else: + raise PolicyValidationError( + '过滤类型只能配置一个,ge 和 le 可以同时使用,', self.data) + return self + + def __call__(self, i): + return self.get_request(i) + + class TencentEipFilter(Filter): schema = None @@ -34,6 +66,7 @@ def __call__(self, i): return False return i + class TencentDiskFilter(Filter): schema = None @@ -45,6 +78,7 @@ def __call__(self, i): return False return i + class TencentCdbFilter(Filter): schema = None @@ -59,6 +93,7 @@ def __call__(self, i): return False return i + class TencentClbFilter(Filter): def validate(self): @@ -67,6 +102,7 @@ def validate(self): def __call__(self, i): return self.get_request(i) + class TencentVpcFilter(Filter): def validate(self): @@ -75,6 +111,7 @@ def validate(self): def __call__(self, i): return self.get_request(i) + class TencentAgeFilter(Filter): """Automatically filter resources older than a given date. @@ -106,7 +143,6 @@ def __call__(self, i): op = OPERATORS[self.data.get('op', 'greater-than')] if not self.threshold_date: - days = self.data.get('days', 0) hours = self.data.get('hours', 0) minutes = self.data.get('minutes', 0) @@ -116,6 +152,7 @@ def __call__(self, i): return op(self.threshold_date, v) + class SGPermission(Filter): """Filter for verifying security group ingress and egress permissions @@ -242,7 +279,7 @@ class SGPermission(Filter): perm_attrs = { 'IpProtocol', "Priority", 'Policy'} filter_attrs = { - 'Cidr', 'CidrV6', 'Ports', 'OnlyPorts', + 'Cidr', 'CidrV6', 'Ports', 'OnlyPorts', 'Action', 'SelfReference', 'Description', 'SGReferences'} attrs = perm_attrs.union(filter_attrs) attrs.add('match-operator') @@ -261,7 +298,7 @@ def process(self, resources, event=None): fattrs = list(sorted(self.perm_attrs.intersection(self.data.keys()))) self.ports = 'Ports' in self.data and self.data['Ports'] or () self.only_ports = ( - 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) + 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) for f in fattrs: fv = self.data.get(f) if isinstance(fv, dict): @@ -275,87 +312,169 @@ def process(self, resources, event=None): return super(SGPermission, self).process(resources, event) - def process_ports(self, perm): - found = None - if self.ip_permissions_type == 'ingress': - poms = perm['IpPermissions']['Ingress'] - else: - poms = perm['IpPermissions']['Egress'] - for ingress in poms: - if ingress['Action'] != 'ACCEPT': - return False - if ingress['Port']: - FromPort = ingress['Port'] + def cidr_process_ports(self, pom, cidr, cidr_key, items): + found = False + accept_drop = pom.get('Action', 'ACCEPT') + action = self.data.get('Action', 'ACCEPT') + if accept_drop == 'ACCEPT': + # 查询ACCEPT 实际ACCEPT + if accept_drop == action: + if pom['Port']: + FromPort = pom['Port'] for port in self.ports: if FromPort == "ALL": - found = True - break + return True else: if ',' in FromPort: strs = FromPort.split(',') for str in strs: if port == int(str): found = True - break + continue elif '-' in FromPort: strs = FromPort.split('-') p1 = int(strs[0]) p2 = int(strs[1]) if port >= p1 and port <= p2: found = True - break + continue else: if port == int(FromPort): found = True - break + continue found = False only_found = False for port in self.only_ports: if port == FromPort: only_found = True if self.only_ports and not only_found: - found = found is None or found and True or False + found = found is False or found and True or False if self.only_ports and only_found: found = False + else: + # 查询DROP 实际ACCEPT + found = False + else: + # 查询DROP 实际DROP + if accept_drop == action: + if pom['Port']: + FromPort = pom['Port'] + for port in self.ports: + if FromPort == "ALL": + return True + else: + if ',' in FromPort: + strs = FromPort.split(',') + for str in strs: + if port == int(str): + return True + elif '-' in FromPort: + strs = FromPort.split('-') + p1 = int(strs[0]) + p2 = int(strs[1]) + if port >= p1 and port <= p2: + return True + else: + if port == int(FromPort): + return True + found = False + only_found = False + for port in self.only_ports: + if port == FromPort: + only_found = True + if self.only_ports and not only_found: + found = found is False or found and True or False + if self.only_ports and only_found: + found = False + else: + # 查询ACCEPT 实际DROP + # 0.0.0.0/0 + CidrBlock = pom.get(cidr, "") + if CidrBlock != self.data.get(cidr_key, ""): + return False + IpProtocol = self.data.get('IpProtocol', "").upper() + if IpProtocol in ["-1", -1]: + IpProtocol = "ALL" + outProtocol = pom.get("Protocol", "").upper() + if outProtocol == "ALL" or IpProtocol == "ALL": + found = True + else: + if IpProtocol != outProtocol: + return False + if pom['Port']: + dropPort = pom['Port'] + for port in self.ports: + if dropPort == "ALL": + return True + else: + if ',' in dropPort: + strs = dropPort.split(',') + for str in strs: + if port == int(str): + self.ports.remove(port) + elif '-' in dropPort: + strs = dropPort.split('-') + p1 = int(strs[0]) + p2 = int(strs[1]) + if port >= p1 and port <= p2: + self.ports.remove(port) + else: + if port == int(dropPort): + self.ports.remove(port) + found = False + only_found = False + for port in self.only_ports: + if port == dropPort: + only_found = True + if self.only_ports and not only_found: + found = found is False or found and True or False + if self.only_ports and only_found: + return False return found - - def _process_cidr(self, cidr_key, cidr_type, SourceCidrIp, perm): - found = None - if not SourceCidrIp: + def _process_cidr(self, cidr_key, cidr_type, cidr, perm): + found = False + if not cidr: return False if self.ip_permissions_type == 'ingress': - items = perm.get('IpPermissions').get('Ingress') + items = perm.get('IpPermissions', {}).get('Ingress', []) else: - items = perm.get('IpPermissions').get('Egress') - for str in items: - SourceCidrIp = str.get(SourceCidrIp) - if SourceCidrIp: - sci = {cidr_type: SourceCidrIp} - match_range = self.data[cidr_key] - if isinstance(match_range, dict): - match_range['key'] = cidr_type - else: - match_range = {cidr_type: match_range} - vf = ValueFilter(match_range, self.manager) - vf.annotate = False - found = vf(sci) - if found: - pass + items = perm.get('IpPermissions', {}).get('Egress', []) + for ip_Permission in items: + # 0.0.0.0/0 + CidrBlock = ip_Permission.get(cidr, "") + if CidrBlock == self.data.get(cidr_key, ""): + found = True + else: + found = False + continue + IpProtocol = self.data.get('IpProtocol', "").upper() + if IpProtocol in ["-1", -1]: + IpProtocol = "ALL" + outProtocol = ip_Permission.get("Protocol", "").upper() + if outProtocol == "ALL" or IpProtocol == "ALL": + found = True + else: + if IpProtocol == outProtocol: + found = True else: found = False + continue + found = self.cidr_process_ports(ip_Permission, cidr, cidr_key, items) + if found: + break return found - def process_cidrs(self, perm, ipv4Cidr, ipv6Cidr): - found_v6 = found_v4 = None + def process_cidrs(self, perm, CidrBlock, Ipv6CidrBlock): + found_v6 = found_v4 = False if 'CidrV6' in self.data: - found_v6 = self._process_cidr('CidrV6', 'CidrIpv6', ipv6Cidr, perm) + found_v6 = self._process_cidr('CidrV6', 'CidrIpv6', Ipv6CidrBlock, perm) if 'Cidr' in self.data: - found_v4 = self._process_cidr('Cidr', 'CidrIp', ipv4Cidr, perm) + found_v4 = self._process_cidr('Cidr', 'CidrIp', CidrBlock, perm) match_op = self.data.get('match-operator', 'and') == 'and' and all or any - cidr_match = [k for k in (found_v6, found_v4) if k is not None] + cidr_match = [k for k in (found_v6, found_v4) if k is not False] if not cidr_match: - return None + return False return match_op(cidr_match) def process_description(self, perm): @@ -391,7 +510,7 @@ def process_self_reference(self, perm, sg_id): def process_sg_references(self, perm, owner_id): sg_refs = self.data.get('SGReferences') if not sg_refs: - return None + return False sg_perm = perm.get('UserIdGroupPairs', []) if not sg_perm: @@ -408,27 +527,34 @@ def process_sg_references(self, perm, owner_id): return False def __call__(self, resource): - result = self.securityGroupAttributeRequst(resource) + perm = self.securityGroupAttributeRequst(resource) matched = [] match_op = self.data.get('match-operator', 'and') == 'and' and all or any - for perm in jmespath.search(self.ip_permissions_key, json.loads(result)): - if perm.get('IpPermissions') is None or len(perm.get('IpPermissions').get(self.direction)) == 0: - continue - perm_matches = {} - perm_matches['ports'] = self.process_ports(perm) - perm_matches['cidrs'] = self.process_self_cidrs(perm) - perm_match_values = list(filter( - lambda x: x is not None, perm_matches.values())) - # account for one python behavior any([]) == False, all([]) == True - if match_op == all and not perm_match_values: - continue + if len(perm.get('IpPermissions', {}).get(self.direction, [])) == 0: + return False + # result = self.securityGroupAttributeRequst(resource) + # matched = [] + # match_op = self.data.get('match-operator', 'and') == 'and' and all or any + # for perm in jmespath.search(self.ip_permissions_key, json.loads(result)): + # if perm.get('IpPermissions') is None or len(perm.get('IpPermissions', {}).get(self.direction, [])) == 0: + # continue + perm_matches = {} + # 将cidrs和ports合并，关联判断 + perm_matches['cidrs'] = self.process_self_cidrs(perm) + return perm_matches['cidrs'] + # perm_match_values = list(filter( + # lambda x: x is not None, perm_matches.values())) + # # account for one python behavior any([]) == False, all([]) == True + # if match_op == all and not perm_match_values: + # return False + # match = match_op(perm_match_values) + # if match: + # matched.append(perm) + # + # if matched: + # resource['Matched%s' % self.ip_permissions_key] = matched + # return True - match = match_op(perm_match_values) - if match: - matched.append(perm) - if matched: - resource['Matched%s' % self.ip_permissions_key] = matched - return True class MetricsFilter(Filter): """Supports metrics filters on resources. @@ -489,7 +615,7 @@ def process(self, resources, event=None): self.statistics = self.data.get('statistics', 'Average') self.model = self.manager.get_model() self.op = OPERATORS[self.data.get('op', 'less-than')] - self.value = self.data['value'] + self.value = self.data.get('value', '') self.namespace = self.model.namespace self.log.debug("Querying metrics for %d", len(resources)) matched = [] @@ -537,15 +663,13 @@ def process_resource_set(self, resource_set): collected_metrics = r.setdefault('c7n_tencent.metrics', {}) key = "%s.%s.%s" % (self.namespace, self.metric, self.statistics) - # print(client.do_action(request)) if key not in collected_metrics: - collected_metrics[key] = json.loads(reponse)["DataPoints"] if len(collected_metrics[key]) == 0: if 'missing-value' not in self.data: continue - collected_metrics[key].append({'timestamp': self.start, self.statistics: self.data['missing-value'], 'c7n_tencent:detail': 'Fill value for missing data'}) - print(collected_metrics[key][0]['Values']) + collected_metrics[key].append({'timestamp': self.start, self.statistics: self.data['missing-value'], + 'c7n_tencent:detail': 'Fill value for missing data'}) if self.data.get('percent-attr', None) != None: rvalue = r[self.data.get('percent-attr')] if self.data.get('attr-multiplier'): @@ -558,6 +682,7 @@ def process_resource_set(self, resource_set): matched.append(r) return matched + SGPermissionSchema = { 'match-operator': {'type': 'string', 'enum': ['or', 'and']}, 'Ports': {'type': 'array', 'items': {'type': 'integer'}}, @@ -565,10 +690,11 @@ def process_resource_set(self, resource_set): 'Policy': {}, 'IpProtocol': { 'oneOf': [ - {'enum': ["-1", -1, 'TCP', 'UDP', 'ICMP', 'ICMPV6']}, + {'enum': ["-1", -1, 'TCP', 'UDP', 'ICMP', 'ICMPV6', 'tcp', 'udp', 'icmp', 'icmpv6']}, {'$ref': '#/definitions/filters/value'} ] }, + 'Action': {'type': 'string', 'enum': ['ACCEPT', 'DROP']}, 'FromPort': {'oneOf': [ {'$ref': '#/definitions/filters/value'}, {'type': 'integer'}]}, diff --git a/tools/c7n_tencent/c7n_tencent/page.py b/tools/c7n_tencent/c7n_tencent/page.py new file mode 100644 index 00000000000..50721140a4e --- /dev/null +++ b/tools/c7n_tencent/c7n_tencent/page.py @@ -0,0 +1,80 @@ +import json +from c7n_tencent.client import Session +from tencentcloud.postgres.v20170312 import postgres_client, models + + +def merge_response(old_response, response_fist_field_name, new_response): + """ + 合并分页返回值 + @param old_response: 合并对象 + @param response_fist_field_name: 需要合并的字段 + @param new_response: 新的对象 + @return: + """ + if not old_response: + return new_response + setattr(old_response, response_fist_field_name, + getattr(old_response, response_fist_field_name) + getattr(new_response, response_fist_field_name)) + return old_response + + +def def_response_len_is_next_fun(old_response, new_response, response_fist_field_name, limit, total_count): + if (len(getattr(old_response, response_fist_field_name)) if old_response else 0 + len( + getattr(new_response, response_fist_field_name)) if new_response else 0) == getattr(new_response, + total_count): + return False + return True + + +def def_is_next_fun(old_response, new_response, response_fist_field_name, limit=20, total_count_field=None): + """ + 默认判断是否有下一页方法 + @param old_response: 合并的response + @param new_response: 新的 response + @param response_fist_field_name: 需要合并的字段 + @param limit: 每页长度 + @param total_count_field: 请求结果中 (总条数)total_count字段名 + @return: + """ + if total_count_field: + if not def_response_len_is_next_fun(old_response, new_response, response_fist_field_name, limit, + total_count_field): + return False + if len(getattr(new_response, response_fist_field_name)) == limit: + return True + return False + + +def page_all(list_func, req, reponse_set_field, total_count_field=None, offset=0, limit=20, old_response=None, + is_next_fun=def_is_next_fun): + """ + 分页查询所有数据 + @param list_func: 查询分页函数 + @param reponse_set_field 返回参数数组字段名称 + @param total_count_field 返回总数名称 + @param req 请求参数 + @param offset: 偏移量 + @param limit: 每页长度 + @param old_response: 合并的response + @param is_next_fun: 判断是否有下一页 + @return: + """ + + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + respose = list_func(req) + old_response = merge_response(old_response, reponse_set_field, respose) + if is_next_fun(old_response, respose, reponse_set_field, limit, total_count_field): + return page_all(list_func, offset + 1, limit, old_response, is_next_fun) + return old_response + + +if __name__ == '__main__': + lsb = Session('AKID65nHX08TO3UZWPPtMchfZEdmrj3A9iHi', 'MXVF2iDZbOzNYVII5nsjtpvBCVZPzbaX', 'ap-shanghai').client( + 'postgres_client') + req = models.DescribeDBInstancesRequest() + res = page_all(lsb.DescribeDBInstances, req, 'DBInstanceSet', 'TotalCount'); + print(res) diff --git a/tools/c7n_tencent/c7n_tencent/query.py b/tools/c7n_tencent/c7n_tencent/query.py index 7c7fd0441f2..2014bcb0fab 100644 --- a/tools/c7n_tencent/c7n_tencent/query.py +++ b/tools/c7n_tencent/c7n_tencent/query.py @@ -41,23 +41,16 @@ def filter(self, resource_manager, **params): params.update(extra_args) if m.service == 'cos': - result = client.listBuckets() + result = client.list_buckets() buckets = [] for b in result.buckets: b.__dict__['F2CId'] = b.__dict__[m.id] buckets.append(b.__dict__) return buckets else: - request = resource_manager.get_request() - if request: - result = request - else: - return None + res = resource_manager.get_request() false = "false" true = "true" - if path is None: - return result - res = jmespath.search(path, eval(result)) for data in res: data['F2CId'] = data[m.id] return res diff --git a/tools/c7n_tencent/c7n_tencent/resources/cdb.py b/tools/c7n_tencent/c7n_tencent/resources/cdb.py index 7b55b25fcdc..c2835adb8bc 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/cdb.py +++ b/tools/c7n_tencent/c7n_tencent/resources/cdb.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.cdb.v20170320 import models from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException @@ -34,20 +36,34 @@ class resource_type(TypeInfo): id = 'InstanceId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeDBInstancesRequest() - resp = Session.client(self, service).DescribeDBInstances(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + while 0 <= offset: + req = models.DescribeDBInstancesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeDBInstances(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('Items', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Cdb.filter_registry.register('Internet') class TencentCdbFilter(TencentCdbFilter): @@ -102,11 +118,11 @@ class InternetAccessCdbFilter(TencentFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if self.data['value']: - if i['WanStatus'] == 1: + if self.data.get('value', ''): + if i.get('WanStatus', '') == 1: return i else: - if i['WanStatus'] != 1: + if i.get('WanStatus', '') != 1: return i return False @@ -130,7 +146,7 @@ class DeviceTypeCdbFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if i['DeviceType'] == self.data['value']: + if i.get('DeviceType', '') == self.data.get('value', ''): return i return False @@ -155,12 +171,12 @@ class AvailablezonesCdbFilter(TencentFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if self.data['value']: - if i['DeployMode'] == 1: + if self.data.get('value', ''): + if i.get('DeployMode', '') == 1: return i return False else: - if i['DeployMode'] != 1: + if i.get('DeployMode', '') != 1: return i return False @@ -183,12 +199,12 @@ class NetworkTypeCdbFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == "vpc": - if i['VpcId']: + if self.data.get('value', '') == "vpc": + if i.get('VpcId', ''): return False return i else: - if i['VpcId']: + if i.get('VpcId', ''): return i return False diff --git a/tools/c7n_tencent/c7n_tencent/resources/clb.py b/tools/c7n_tencent/c7n_tencent/resources/clb.py index 1b7883cf91a..29f9171fa2d 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/clb.py +++ b/tools/c7n_tencent/c7n_tencent/resources/clb.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.clb.v20180317 import models from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException @@ -33,23 +35,35 @@ class resource_type(TypeInfo): id = 'LoadBalancerId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - # 实例化一个cvm实例信息查询请求对象,每个接口都会对应一个request对象。 - req = models.DescribeLoadBalancersRequest() - params = '{}' - req.from_json_string(params) - resp = Session.client(self, service).DescribeLoadBalancers(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + while 0 <= offset: + # 实例化一个cvm实例信息查询请求对象,每个接口都会对应一个request对象。 + req = models.DescribeLoadBalancersRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeLoadBalancers(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('LoadBalancerSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Clb.filter_registry.register('unused') @@ -69,7 +83,7 @@ class TencentClbFilter(TencentClbFilter): schema = type_schema('unused') def get_request(self, i): - LoadBalancerId = i['LoadBalancerId'] + LoadBalancerId = i.get('LoadBalancerId', '') # clb 查询clb下是否有监听 self.LoadBalancerId = LoadBalancerId req = models.DescribeTargetsRequest() @@ -103,7 +117,7 @@ class AclsClbFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if i['LoadBalancerType'] and i['LoadBalancerType'] == self.data['value']: + if i.get('LoadBalancerType', '') and i.get('LoadBalancerType', '') == self.data.get('value', ''): return False return i @@ -126,8 +140,8 @@ class BandwidthClbFilter(TencentFilter): **{'value': {'type': 'number'}}) def get_request(self, i): - InternetMaxBandwidthOut = i['NetworkAttributes']['InternetMaxBandwidthOut'] - if InternetMaxBandwidthOut and self.data['value'] < InternetMaxBandwidthOut: + InternetMaxBandwidthOut = i.get('NetworkAttributes', {}).get('InternetMaxBandwidthOut', 0) + if InternetMaxBandwidthOut and self.data.get('value', '') < InternetMaxBandwidthOut: return False return i @@ -151,12 +165,12 @@ class NetworkTypeClbFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == "vpc": - if i['VpcId']: + if self.data.get('value', '') == "vpc": + if i.get('VpcId', ''): return False return i else: - if i['VpcId']: + if i.get('VpcId', ''): return i return False diff --git a/tools/c7n_tencent/c7n_tencent/resources/cos.py b/tools/c7n_tencent/c7n_tencent/resources/cos.py index f6fc901c8c0..cd1433fcd68 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/cos.py +++ b/tools/c7n_tencent/c7n_tencent/resources/cos.py @@ -40,15 +40,20 @@ class resource_type(TypeInfo): id = 'Name' def get_request(self): + resp = [] try: resp = Session.client(self, service).list_buckets() _resp_ = [] - for i in resp['Buckets']['Bucket']: + Buckets = resp.get("Buckets", {}) + if Buckets is None: + Buckets = {} + for i in Buckets.get("Bucket", []): if i['Location'] == regionId: _resp_.append(i) + resp['Buckets'] = Buckets resp['Buckets']['Bucket'] = _resp_ - for obj in resp['Buckets']['Bucket']: - if regionId != obj['Location']: + for obj in Buckets.get("Bucket", []): + if regionId != obj.get('Location', ''): continue objects = list() try: @@ -59,16 +64,14 @@ def get_request(self): continue objects.append(response['Contents']) #响应条目是否被截断，布尔值，例如true或false - if response['IsTruncated'] == 'false': + if response.get('IsTruncated', '') == 'false': continue obj['Objects'] = objects except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 logging.error(str(e)) - return json.dumps(resp) except TencentCloudSDKException as err: logging.error(err) - return json.dumps(resp) - return json.dumps(resp) + return eval(json.dumps(Buckets.get('Bucket', []))) @@ -105,14 +108,14 @@ def process_bucket(self, b): response = Session.client(self, service).get_bucket_acl( Bucket=b['Name'] ) - Grant = response.get('AccessControlList').get('Grant') + Grant = response.get('AccessControlList', {}).get('Grant', []) for i in Grant: # 指明授予被授权者的存储桶权限，可选值有 FULL_CONTROL，WRITE，READ，分别对应读写权限、写权限、读权限 - if i.get('Permission') == 'FULL_CONTROL': + if i.get('Permission', '') == 'FULL_CONTROL': b['Permission'] = response return b else: - if self.data['value'] in i.get('Permission'): + if self.data.get('value', '') in i.get('Permission', ''): b['Permission'] = response return b return False @@ -139,7 +142,7 @@ def get_request(self, i): result = Session.client(self, service).get_bucket_referer( Bucket=i['Name'] ) - if self.data['value']: + if self.data.get('value', ''): if result and result.Status == 'Eabled': return False else: @@ -173,7 +176,7 @@ def get_request(self, i): result = Session.client(self, service).get_bucket_encryption( Bucket=i['Name'] ) - if self.data['value']: + if self.data.get('value', ''): if 'Error' in result: return i else: diff --git a/tools/c7n_tencent/c7n_tencent/resources/cvm.py b/tools/c7n_tencent/c7n_tencent/resources/cvm.py index ec31c2dc51f..e32fe2bcea6 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/cvm.py +++ b/tools/c7n_tencent/c7n_tencent/resources/cvm.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import operator +import jmespath from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException # 导入对应产品模块的client models。 from tencentcloud.cvm.v20170312 import models @@ -39,24 +41,38 @@ class resource_type(TypeInfo): dimension = 'InstanceId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - # 实例化一个cvm实例信息查询请求对象,每个接口都会对应一个request对象。 - req = models.DescribeInstancesRequest() - # 通过client对象调用DescribeInstances方法发起请求。注意请求方法名与请求对象是对应的。 - # 返回的resp是一个DescribeInstancesResponse类的实例，与请求对象对应。 - resp = Session.client(self, service).DescribeInstances(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.InstanceSet) - # print(resp.to_json_string()) + while 0 <= offset: + # 实例化一个cvm实例信息查询请求对象,每个接口都会对应一个request对象。 + req = models.DescribeInstancesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + # 通过client对象调用DescribeInstances方法发起请求。注意请求方法名与请求对象是对应的。 + # 返回的resp是一个DescribeInstancesResponse类的实例，与请求对象对应。 + resp = Session.client(self, service).DescribeInstances(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('InstanceSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.InstanceSet) + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Cvm.filter_registry.register('metrics') class CvmMetricsFilter(MetricsFilter): @@ -94,7 +110,7 @@ class CvmAgeFilter(TencentAgeFilter): def get_resource_date(self, i): # '2019-11-20T08:21:02Z' - return i['CreatedTime'] + return i.get('CreatedTime', '2021-08-10T08:21:02Z') @Cvm.action_registry.register('start') class Start(MethodAction): @@ -176,7 +192,7 @@ class PublicIpAddress(TencentFilter): def get_request(self, i): data = i[self.public_ip_address] - if len(data) == 0: + if data is None or len(data) == 0: return False return i @@ -209,6 +225,6 @@ class StopChargingMode(TencentFilter): def get_request(self, i): data = i[self.stop_charging_mode] - if data == self.data['value']: + if data == self.data.get('value', ''): return False return i diff --git a/tools/c7n_tencent/c7n_tencent/resources/dcdb.py b/tools/c7n_tencent/c7n_tencent/resources/dcdb.py index f244268f0b0..d3000fff353 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/dcdb.py +++ b/tools/c7n_tencent/c7n_tencent/resources/dcdb.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.dcdb.v20180411 import models @@ -33,20 +35,34 @@ class resource_type(TypeInfo): id = 'InstancesId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeDCDBInstancesRequest() - resp = Session.client(self, service).DescribeDCDBInstances(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) + while 0 <= offset: + req = models.DescribeDCDBInstancesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeDCDBInstances(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('Instances', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Dcdb.filter_registry.register('Internet') class TencentDcdbFilter(TencentCdbFilter): diff --git a/tools/c7n_tencent/c7n_tencent/resources/disk.py b/tools/c7n_tencent/c7n_tencent/resources/disk.py index a377badd710..ddcf7cc50b6 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/disk.py +++ b/tools/c7n_tencent/c7n_tencent/resources/disk.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.cbs.v20170312 import models from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException @@ -34,20 +36,34 @@ class resource_type(TypeInfo): id = 'DiskId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeDisksRequest() - resp = Session.client(self, service).DescribeDisks(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + while 0 <= offset: + req = models.DescribeDisksRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeDisks(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('DiskSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Disk.filter_registry.register('unused') class TencentDiskFilter(TencentDiskFilter): @@ -97,7 +113,7 @@ class encrypt(TencentFilter): def get_request(self, i): data = i[self.encrypt] - if data == self.data['value']: + if data == self.data.get('value', ''): return False return i diff --git a/tools/c7n_tencent/c7n_tencent/resources/eip.py b/tools/c7n_tencent/c7n_tencent/resources/eip.py index 8b70649a95e..4f67a6e557a 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/eip.py +++ b/tools/c7n_tencent/c7n_tencent/resources/eip.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.vpc.v20170312 import models @@ -34,20 +36,34 @@ class resource_type(TypeInfo): id = 'AddressId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeAddressesRequest() - resp = Session.client(self, service).DescribeAddresses(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + while 0 <= offset: + req = models.DescribeAddressesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeAddresses(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('AddressSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Eip.filter_registry.register('unused') class TencentEipFilter(TencentEipFilter): @@ -87,7 +103,7 @@ class BandwidthEipFilter(TencentFilter): **{'value': {'type': 'number'}}) def get_request(self, i): - if i['Bandwidth'] and self.data['value'] < i['Bandwidth']: + if i.get('Bandwidth', '') and self.data.get('value', '') < i.get('Bandwidth', ''): return False return i diff --git a/tools/c7n_tencent/c7n_tencent/resources/es.py b/tools/c7n_tencent/c7n_tencent/resources/es.py new file mode 100644 index 00000000000..bc82cd394e7 --- /dev/null +++ b/tools/c7n_tencent/c7n_tencent/resources/es.py @@ -0,0 +1,634 @@ +import jmespath +from c7n_tencent import page, filter_util +from c7n_tencent.client import Session +from c7n_tencent.filters.filter import TencentFilter, TencentEsFilter +from c7n_tencent.provider import resources +from c7n_tencent.query import QueryResourceManager, TypeInfo +from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException +from tencentcloud.es.v20180416 import es_client, models +import logging + +from c7n.utils import type_schema + +service = 'es_client.es' + + +@resources.register('es') +class Postgres(QueryResourceManager): + class resource_type(TypeInfo): + service = 'postgres_client.postgres' + enum_spec = (None, 'InstanceList', None) + id = 'InstanceId' + + def get_request(self): + listField = self.resource_type.enum_spec[1] + res = [] + try: + req = models.DescribeInstancesRequest() + client = Session.client(self, service) + # 查询到所有分页数据 + resp = page.page_all(client.DescribeInstances, req, listField, + 'TotalCount') + # 将结果转换为字典 + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + res = jmespath.search(listField, eval(respose)) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + except TencentCloudSDKException as err: + logging.error(err) + return res + + +@Postgres.filter_registry.register('InstanceId') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'InstanceId', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('InstanceName') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'InstanceName', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('Region') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'Region', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('Zone') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'Zone', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('AppId') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'AppId', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('Uin') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'Uin', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('VpcUid') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'VpcUid', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('SubnetUid') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'SubnetUid', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('Status') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'Status', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('ChargeType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'ChargeType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('ChargePeriod') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'ChargePeriod', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('RenewFlag') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'RenewFlag', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('CpuNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'CpuNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('MemSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MemSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('DiskType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'DiskType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('DiskSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'DiskSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('EsDomain') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsDomain', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('EsVip') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsVip', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('EsPort') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsPort', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('KibanaUrl') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'KibanaUrl', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('EsVersion') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsVersion', + **filter_util.get_schema('string')) + + +# @Postgres.filter_registry.register('EsConfig') +# class NetworkTypePostgresFilter(TencentEsFilter): +# schema = type_schema( +# 'EsConfig', +# **filter_util.get_schema('string')) +# + +@Postgres.filter_registry.register('EsAcl.BlackIpList') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsAcl.BlackIpList', + **filter_util.get_schema('list_string')) + + +@Postgres.filter_registry.register('EsAcl.WhiteIpList') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsAcl.WhiteIpList', + **filter_util.get_schema('list_string')) + + +@Postgres.filter_registry.register('CreateTime') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'CreateTime', + **filter_util.get_schema('time')) + + +@Postgres.filter_registry.register('UpdateTime') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'UpdateTime', + **filter_util.get_schema('time')) + + +@Postgres.filter_registry.register('Deadline') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'Deadline', + **filter_util.get_schema('time')) + + +@Postgres.filter_registry.register('InstanceType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'InstanceType', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('IkConfig.MainDict..Key') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig..MainDict.Key', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.MainDict..Name') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig..MainDict.Name', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.MainDict..Size') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig..MainDict.Size', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('IkConfig.Stopwords..Key') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig..Stopwords.Key', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.Stopwords..Name') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig..Stopwords.Name', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.Stopwords..Size') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.Stopwords..Size', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('IkConfig.QQDict..Key') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.QQDict..Key', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.QQDict..Name') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.QQDict..Name', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.QQDict..Size') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.QQDict..Size', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('IkConfig.Synonym..Key') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.Synonym..Key', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.Synonym..Name') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.Synonym..Name', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('IkConfig.Synonym..Size') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.Synonym..Size', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('IkConfig.UpdateType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'IkConfig.UpdateType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('MasterNodeInfo.EnableDedicatedMaster') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.UpdateType', + **filter_util.get_schema('boolean')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeCpuNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeCpuNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeMemSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeMemSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeDiskSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeDiskSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('MasterNodeInfo.MasterNodeDiskType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MasterNodeInfo.MasterNodeDiskType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('CosBackup.IsAutoBackup') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'CosBackup.IsAutoBackup', + **filter_util.get_schema('boolean')) + + +@Postgres.filter_registry.register('CosBackup.BackupTime') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'CosBackup.BackupTime', + **filter_util.get_schema('time')) + + +@Postgres.filter_registry.register('AllowCosBackup') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'AllowCosBackup', + **filter_util.get_schema('boolean')) + + +@Postgres.filter_registry.register('TagList') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'TagList', + **filter_util.get_schema('list_string')) + + +@Postgres.filter_registry.register('LicenseType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'LicenseType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('EnableHotWarmMode') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EnableHotWarmMode', + **filter_util.get_schema('boolean')) + + +@Postgres.filter_registry.register('WarmNodeType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmNodeType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('WarmNodeNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmNodeNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('WarmCpuNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmCpuNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('WarmMemSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmMemSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('WarmDiskType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmDiskType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('WarmDiskSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'WarmDiskSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..NodeNum') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..NodeNum', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..NodeType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..NodeType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeInfoList..Type') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..Type', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeInfoList..DiskType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..DiskType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeInfoList..DiskSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..DiskSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..LocalDiskInfo') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..LocalDiskInfo', + **filter_util.get_schema('is_empty')) + + +@Postgres.filter_registry.register('NodeInfoList.LocalDiskInfo..LocalDiskType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList.LocalDiskInfo..LocalDiskType', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('NodeInfoList.LocalDiskInfo..LocalDiskSize') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList.LocalDiskInfo..LocalDiskSize', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList.LocalDiskInfo..LocalDiskCount') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList.LocalDiskInfo..LocalDiskCount', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..DiskCount') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..DiskCount', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..DiskEncrypt') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..DiskEncrypt', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('NodeInfoList..DiskEncrypt') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'NodeInfoList..DiskEncrypt', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('EsPublicUrl') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsPublicUrl', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('MultiZoneInfo..Zone') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MultiZoneInfo..Zone', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('MultiZoneInfo..SubnetId') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'MultiZoneInfo..SubnetId', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('DeployMode') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'DeployMode', + **filter_util.get_schema('number')) + + +@Postgres.filter_registry.register('PublicAccess') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'PublicAccess', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('EsPublicAcl.BlackIpList') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsPublicAcl.BlackIpList', + **filter_util.get_schema('list_string')) + + +@Postgres.filter_registry.register('EsPublicAcl.WhiteIpList') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'EsPublicAcl.BlackIpList', + **filter_util.get_schema('list_string')) + + +@Postgres.filter_registry.register('KibanaPrivateUrl') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'KibanaPrivateUrl', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('KibanaPublicAccess') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'KibanaPublicAccess', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('KibanaPrivateAccess') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'KibanaPrivateAccess', + **filter_util.get_schema('string')) + + +@Postgres.filter_registry.register('SecurityType') +class NetworkTypePostgresFilter(TencentEsFilter): + schema = type_schema( + 'SecurityType', + **filter_util.get_schema('number')) diff --git a/tools/c7n_tencent/c7n_tencent/resources/mongodb.py b/tools/c7n_tencent/c7n_tencent/resources/mongodb.py index 7f8d897148b..f4a1a8470d9 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/mongodb.py +++ b/tools/c7n_tencent/c7n_tencent/resources/mongodb.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.mongodb.v20190725 import models @@ -33,20 +35,34 @@ class resource_type(TypeInfo): id = 'InstanceId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeDBInstancesRequest() - resp = Session.client(self, service).DescribeDBInstances(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) + while 0 <= offset: + req = models.DescribeDBInstancesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeDBInstances(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('InstanceDetails', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @MongoDB.filter_registry.register('network-type') class NetworkTypeMongoDBFilter(TencentFilter): @@ -67,12 +83,12 @@ class NetworkTypeMongoDBFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == "vpc": - if i['VpcId']: + if self.data.get('value', '') == "vpc": + if i.get('VpcId', ''): return False return i else: - if i['VpcId']: + if i.get('VpcId', ''): return i return False @@ -97,10 +113,10 @@ class InternetAccessMongoDBFilter(TencentFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if self.data['value']: - if i['NetType'] == 0: + if self.data.get('value', ''): + if i.get('NetType', 0) == 0: return i else: - if i['NetType'] != 0: + if i.get('NetType', 0) != 0: return i return False \ No newline at end of file diff --git a/tools/c7n_tencent/c7n_tencent/resources/postgres.py b/tools/c7n_tencent/c7n_tencent/resources/postgres.py new file mode 100644 index 00000000000..f3c55ed71a7 --- /dev/null +++ b/tools/c7n_tencent/c7n_tencent/resources/postgres.py @@ -0,0 +1,91 @@ +import jmespath +from c7n_tencent import page +from c7n_tencent.query import QueryResourceManager, TypeInfo +from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException +import logging +from c7n.utils import type_schema +from c7n_tencent.client import Session +from c7n_tencent.filters.filter import TencentFilter +from c7n_tencent.provider import resources +from tencentcloud.postgres.v20170312 import models + +service = 'postgres_client.postgres' + + +@resources.register('postgres') +class Postgres(QueryResourceManager): + class resource_type(TypeInfo): + service = 'postgres_client.postgres' + enum_spec = (None, 'DBInstanceSet', None) + id = 'DBInstanceId' + + def get_request(self): + res = [] + try: + req = models.DescribeDBInstancesRequest() + client = Session.client(service) + # 查询到所有分页数据 + resp = page.page_all(client.DescribeDBInstances, req, 'DBInstanceSet', + 'TotalCount') + # 将结果转换为字典 + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + res = jmespath.search('DBInstanceSet', eval(respose)) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + except TencentCloudSDKException as err: + logging.error(err) + return res + + +@Postgres.filter_registry.register('network-type') +class NetworkTypePostgresFilter(TencentFilter): + schema = type_schema( + 'network-type', + **{'value': {'type': 'string'}}) + + def get_request(self, i): + if self.data.get('value', '') == "vpc": + if i.get('VpcId', ''): + return False + return i + else: + if i.get('VpcId', ''): + return i + return False + + +def network_is_public(obj): + return obj and obj['NetType'] and obj['NetType'] == 'public' and obj['Status'] and obj['Status'] == 'opened' + + +@Postgres.filter_registry.register('internet-access') +class InternetAccessTypePostgresFilter(TencentFilter): + schema = type_schema( + 'internet-access', + **{'value': {'type': 'boolean'}}) + + """ + Filters + :Example: + .. code-block:: yaml + + policies: + # 检测您账号下postgres实例不允许任意来源公网访问，视为“合规” + - name: tencent-postgres-internet-access + resource: tencent.postgres + filters: + - type: internet-access + value: true + """ + + def get_request(self, i): + if self.data.get('value', ''): + if len(list(filter(network_is_public, i['DBInstanceNetInfo']))) > 0: + return i + else: + return False + else: + if len(list(filter(network_is_public, i['DBInstanceNetInfo']))) == 0: + return i + else: + return False diff --git a/tools/c7n_tencent/c7n_tencent/resources/redis.py b/tools/c7n_tencent/c7n_tencent/resources/redis.py index a1db35cb84b..627d2a4bc5c 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/redis.py +++ b/tools/c7n_tencent/c7n_tencent/resources/redis.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.redis.v20180412 import models from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException @@ -33,20 +35,34 @@ class resource_type(TypeInfo): id = 'InstanceId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeInstancesRequest() - resp = Session.client(self, service).DescribeInstances(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) + while 0 <= offset: + req = models.DescribeInstancesRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeInstances(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('InstanceSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Redis.filter_registry.register('network-type') class NetworkTypeRedisFilter(TencentFilter): @@ -67,12 +83,12 @@ class NetworkTypeRedisFilter(TencentFilter): **{'value': {'type': 'string'}}) def get_request(self, i): - if self.data['value'] == "vpc": - if i['VpcId']: + if self.data.get('value', '') == "vpc": + if i.get('VpcId', ''): return False return i else: - if i['VpcId']: + if i.get('VpcId', ''): return i return False @@ -97,10 +113,10 @@ class InternetAccessRedisFilter(TencentFilter): **{'value': {'type': 'boolean'}}) def get_request(self, i): - if self.data['value']: - if i['NetType'] == 0: + if self.data.get('value', ''): + if i.get('NetType', '') == 0: return i else: - if i['NetType'] != 0: + if i.get('NetType', '') != 0: return i return False \ No newline at end of file diff --git a/tools/c7n_tencent/c7n_tencent/resources/resource_map.py b/tools/c7n_tencent/c7n_tencent/resources/resource_map.py index fa81f586896..d86b446103f 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/resource_map.py +++ b/tools/c7n_tencent/c7n_tencent/resources/resource_map.py @@ -10,5 +10,6 @@ "tencent.redis": "c7n_tencent.resources.redis.Redis", "tencent.mongodb": "c7n_tencent.resources.mongodb.MongoDB", "tencent.security-group": "c7n_tencent.resources.securitygroup.SecurityGroup", - + "tencent.postgres": "c7n_tencent.resources.postgres.Postgres", + "tencent.es": "c7n_tencent.resources.es.es" } diff --git a/tools/c7n_tencent/c7n_tencent/resources/securitygroup.py b/tools/c7n_tencent/c7n_tencent/resources/securitygroup.py index d2e61b2e139..6eac3685a5f 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/securitygroup.py +++ b/tools/c7n_tencent/c7n_tencent/resources/securitygroup.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import jmespath @@ -36,26 +37,41 @@ class resource_type(TypeInfo): id = 'SecurityGroupId' def get_request(self): + # 为什么设置成字符串，不晓得sd可怎么设计的，全靠猜。 + offset = '0' + limit = '20' + res = [] try: - req = models.DescribeSecurityGroupsRequest() - resp = Session.client(self, service).DescribeSecurityGroups(req) - for res in resp.SecurityGroupSet: - req2 = models.DescribeSecurityGroupPoliciesRequest() - params = '{"SecurityGroupId":"' + res.SecurityGroupId + '"}' - req2.from_json_string(params) - resp2 = Session.client(self, service).DescribeSecurityGroupPolicies(req2) - res.IpPermissions = resp2.SecurityGroupPolicySet - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) - - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + while 0 <= int(offset): + req = models.DescribeSecurityGroupsRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeSecurityGroups(req) + for sg in resp.SecurityGroupSet: + req2 = models.DescribeSecurityGroupPoliciesRequest() + params = '{"SecurityGroupId":"' + sg.SecurityGroupId + '"}' + req2.from_json_string(params) + resp2 = Session.client(self, service).DescribeSecurityGroupPolicies(req2) + sg.IpPermissions = resp2.SecurityGroupPolicySet + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('SecurityGroupSet', eval(respose)) + res = res + result + if len(result) == int(limit): + offset = str(int(offset) + len(result)) + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) + + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @SecurityGroup.action_registry.register('delete') @@ -95,13 +111,15 @@ class IPPermission(SGPermission): filters: - or: - type: ingress - IpProtocol: tcp + IpProtocol: "-1" Ports: [20,21,22,25,80,443,465,1433,1434,3306,3389,4333,5432,5500] Cidr: "0.0.0.0/0" + Action: "ACCEPT" - type: ingress - IpProtocol: tcp + IpProtocol: "-1" Ports: [20,21,22,25,80,443,465,1433,1434,3306,3389,4333,5432,5500] CidrV6: "::/0" + Action: "ACCEPT" """ ip_permissions_key = "SecurityGroupSet" ip_permissions_type = "ingress" @@ -113,23 +131,11 @@ class IPPermission(SGPermission): schema['properties'].update(SGPermissionSchema) def process_self_cidrs(self, perm): - self.process_cidrs(perm, 'CidrBlock', 'Ipv6CidrBlock') + return self.process_cidrs(perm, 'CidrBlock', 'Ipv6CidrBlock') def securityGroupAttributeRequst(self, sg): self.direction = 'Ingress' - req = models.DescribeSecurityGroupsRequest() - params = '{"SecurityGroupId" :"' + sg["SecurityGroupId"] + '"}' - req.from_json_string(params) - resp = Session.client(self, service).DescribeSecurityGroups(req) - for res in resp.SecurityGroupSet: - if sg["SecurityGroupId"] != res.SecurityGroupId: - continue - req2 = models.DescribeSecurityGroupPoliciesRequest() - params = '{"SecurityGroupId":"' + res.SecurityGroupId + '"}' - req2.from_json_string(params) - resp2 = Session.client(self, service).DescribeSecurityGroupPolicies(req2) - res.IpPermissions = resp2.SecurityGroupPolicySet - return resp.to_json_string().replace('null', 'None') + return sg @SecurityGroup.filter_registry.register('egress') class IPPermission(SGPermission): @@ -144,20 +150,10 @@ class IPPermission(SGPermission): def securityGroupAttributeRequst(self, sg): self.direction = 'Egress' - req = models.DescribeSecurityGroupsRequest() - params = '{"SecurityGroupId" :"' + sg["SecurityGroupId"] + '"}' - req.from_json_string(params) - resp = Session.client(self, service).DescribeSecurityGroups(req) - for res in resp.SecurityGroupSet: - req2 = models.DescribeSecurityGroupPoliciesRequest() - params = '{"SecurityGroupId":"' + res.SecurityGroupId + '"}' - req2.from_json_string(params) - resp2 = Session.client(self, service).DescribeSecurityGroupPolicies(req2) - res.IpPermissions = resp2.SecurityGroupPolicySet - return resp.to_json_string().replace('null', 'None') + return sg -def process_self_cidrs(self, perm): - self.process_cidrs(perm, "CidrBlock", "Ipv6CidrBlock") + def process_self_cidrs(self, perm): + return self.process_cidrs(perm, "CidrBlock", "Ipv6CidrBlock") @SecurityGroup.filter_registry.register('source-cidr-ip') @@ -183,6 +179,6 @@ class SourceCidrIp(TencentFilter): def get_request(self, sg): for cidr in jmespath.search(self.ip_permissions_key, sg): - if cidr['CidrBlock'] == self.data['value']: + if cidr['CidrBlock'] == self.data.get('value', ''): return sg return False \ No newline at end of file diff --git a/tools/c7n_tencent/c7n_tencent/resources/vpc.py b/tools/c7n_tencent/c7n_tencent/resources/vpc.py index e7f3cdde356..dc562c83c35 100644 --- a/tools/c7n_tencent/c7n_tencent/resources/vpc.py +++ b/tools/c7n_tencent/c7n_tencent/resources/vpc.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging +import jmespath from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.vpc.v20170312 import models @@ -35,22 +37,34 @@ class resource_type(TypeInfo): id = 'VpcId' def get_request(self): + offset = 0 + limit = 20 + res = [] try: - req = models.DescribeVpcsRequest() - params = '{}' - req.from_json_string(params) - resp = Session.client(self, service).DescribeVpcs(req) - # 输出json格式的字符串回包 - # print(resp.to_json_string(indent=2)) + while 0 <= offset: + req = models.DescribeVpcsRequest() + params = { + "Offset": offset, + "Limit": limit + } + req.from_json_string(json.dumps(params)) + resp = Session.client(self, service).DescribeVpcs(req) + respose = resp.to_json_string().replace('null', 'None').replace('false', 'False').replace('true', 'True') + result = jmespath.search('VpcSet', eval(respose)) + res = res + result + if len(result) == limit: + offset += limit + else: + return res + # 输出json格式的字符串回包 + # print(resp.to_json_string(indent=2)) - # 也可以取出单个值。 - # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 - # print(resp.to_json_string()) + # 也可以取出单个值。 + # 你可以通过官网接口文档或跳转到response对象的定义处查看返回字段的定义。 + # print(resp.to_json_string()) except TencentCloudSDKException as err: logging.error(err) - return False - # tencent 返回的json里居然不是None，而是java的null，活久见 - return resp.to_json_string().replace('null', 'None') + return res @Vpc.filter_registry.register('unused') class TencentVpcFilter(TencentVpcFilter): @@ -67,7 +81,7 @@ class TencentVpcFilter(TencentVpcFilter): schema = type_schema('AVAILABLE') def get_request(self, i): - VpcId = i['VpcId'] + VpcId = i.get('VpcId', '') #vpc 查询vpc下是否有ecs资源 cvms = Cvm.get_request(self) cvms_req = eval(cvms.replace('false', 'False'))['InstanceSet'] @@ -80,7 +94,6 @@ def get_request(self, i): clbs_req = eval(clbs.replace('false', 'False'))['LoadBalancerSet'] if clbs_req: for clb in clbs_req: - print(clb['VpcId']) if VpcId == clb['VpcId']: return None return i diff --git a/tools/c7n_tencent/requirements.txt b/tools/c7n_tencent/requirements.txt index 44307b3864a..a444ea984e3 100644 --- a/tools/c7n_tencent/requirements.txt +++ b/tools/c7n_tencent/requirements.txt @@ -1,4 +1,6 @@ pycryptodome==3.9.8 crcmod==1.7 tencentcloud-sdk-python==3.0.234 -cos-python-sdk-v5==1.8.2 \ No newline at end of file +cos-python-sdk-v5==1.8.2 +cryptography==2.9.2 +jsonpath==0.82 \ No newline at end of file diff --git a/tools/c7n_vsphere/.gitignore b/tools/c7n_vsphere/.gitignore new file mode 100644 index 00000000000..1e7bb232c6a --- /dev/null +++ b/tools/c7n_vsphere/.gitignore @@ -0,0 +1,4 @@ +*pyc +*py~ +__pycache__ +*.egg-info diff --git a/tools/c7n_vsphere/c7n_vsphere/__init__.py b/tools/c7n_vsphere/c7n_vsphere/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/actions/__init__.py b/tools/c7n_vsphere/c7n_vsphere/actions/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/actions/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/actions/core.py b/tools/c7n_vsphere/c7n_vsphere/actions/core.py new file mode 100644 index 00000000000..6263705fb9b --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/actions/core.py @@ -0,0 +1,117 @@ +# Copyright 2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from c7n.actions import Action as BaseAction +from c7n.utils import local_session, chunks + + +class Action(BaseAction): + pass + + +class MethodAction(Action): + """Invoke an api call on each resource. + + Quite a number of procedural actions are simply invoking an api + call on a filtered set of resources. The exact handling is mostly + boilerplate at that point following an 80/20 rule. This class is + an encapsulation of the 80%. + """ + + # method we'll be invoking + method_spec = () + + # batch size + chunk_size = 20 + + # implicitly filter resources by state, (attr_name, (valid_enum)) + attr_filter = () + + # error codes that can be safely ignored + ignore_error_codes = () + + permissions = () + method_perm = None + + def validate(self): + if not self.method_spec: + raise NotImplementedError("subclass must define method_spec") + return self + + def filter_resources(self, resources): + rcount = len(resources) + attr_name, valid_enum = self.attr_filter + resources = [r for r in resources if r.get(attr_name) in valid_enum] + if len(resources) != rcount: + self.log.warning( + "policy:%s action:%s implicitly filtered %d resources to %d by attr:%s", + self.manager.ctx.policy.name, + self.type, + rcount, + len(resources), + attr_name, + ) + return resources + + def process(self, resources): + + if self.attr_filter: + resources = self.filter_resources(resources) + model = self.manager.get_model() + session = local_session(self.manager.session_factory) + client = self.get_client(session, model) + for resource_set in chunks(resources, self.chunk_size): + self.process_resource_set(client, model, resource_set) + + def process_resource_set(self, client, model, resources): + result_key = self.method_spec.get('result_key') + annotation_key = self.method_spec.get('annotation_key') + for resource in resources: + requst = self.get_request(resource) + result = self.invoke_api(client, requst) + if result_key and annotation_key: + resource[annotation_key] = result.get(result_key) + + def invoke_api(self, client, requst): + try: + return client.do_action(requst) + except: + raise + + def get_permissions(self): + if self.permissions: + return self.permissions + m = self.manager.resource_type + method = self.method_perm + if not method and 'op' not in self.method_spec: + return () + if not method: + method = self.method_spec['op'] + component = m.component + if '.' in component: + component = component.split('.')[-1] + return ("{}.{}.{}".format( + m.perm_service or m.service, component, method),) + + def get_operation_name(self, model, resource): + return self.method_spec['op'] + + def get_resource_params(self, model, resource): + raise NotImplementedError("subclass responsibility") + + def get_request(self, resource): + raise NotImplementedError("subclass responsibility") + + def get_client(self, session, model): + return session.client(model.service) diff --git a/tools/c7n_vsphere/c7n_vsphere/actions/cscc.py b/tools/c7n_vsphere/c7n_vsphere/actions/cscc.py new file mode 100644 index 00000000000..5160dd67e51 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/actions/cscc.py @@ -0,0 +1,243 @@ +# Copyright 2018-2019 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime +import hashlib +import json +from urllib.parse import urlparse + +from c7n_vsphere.provider import resources as vsphere_resources + +from c7n.exceptions import PolicyExecutionError, PolicyValidationError +from c7n.utils import local_session, type_schema +from .core import MethodAction + + +class PostFinding(MethodAction): + """Post finding for matched resources to Cloud Security Command Center. + + + :Example: + + .. code-block:: yaml + + policies: + - name: vsphere-instances-with-label + resource: vsphere.instance + filters: + - "tag:name": "bad-instance" + actions: + - type: post-finding + org-domain: example.io + category: MEDIUM_INTERNET_SECURITY + + The source for custodian can either be specified inline to the policy, or + custodian can generate one at runtime if it doesn't exist given a org-domain + or org-id. + + Finding updates are not currently supported, due to upstream api issues. + """ + schema = type_schema( + 'post-finding', + **{ + 'source': { + 'type': 'string', + 'description': 'qualified name of source to post to CSCC as'}, + 'org-domain': {'type': 'string'}, + 'org-id': {'type': 'integer'}, + 'category': {'type': 'string'}}) + schema_alias = True + method_spec = {'op': 'create', 'result': 'name', 'annotation_key': 'c7n:Finding'} + + # create throws error if already exists, patch method has bad docs. + ignore_error_codes = (409,) + + CustodianSourceName = 'CloudCustodian' + DefaultCategory = 'Custodian' + Service = 'securitycenter' + ServiceVersion = 'v1beta1' + + _source = None + + # security center permission model is pretty obtuse to correct + permissions = ( + 'securitycenter.findings.list', + 'securitycenter.findings.update', + 'resourcemanager.organizations.get', + 'securitycenter.assetsecuritymarks.update', + 'securitycenter.sources.update', + 'securitycenter.sources.list' + ) + + def validate(self): + if not any([self.data.get(k) for k in ('source', 'org-domain', 'org-id')]): + raise PolicyValidationError( + "policy:%s CSCC post-finding requires one of source, org-domain, org-id" % ( + self.manager.ctx.policy.name)) + + def process(self, resources): + self.initialize_source() + return super(PostFinding, self).process(resources) + + def get_client(self, session, model): + return session.client( + self.Service, self.ServiceVersion, 'organizations.sources.findings') + + def get_resource_params(self, model, resource): + return self.get_finding(resource) + + def initialize_source(self): + # Ideally we'll be given a source, but we'll attempt to auto create it + # if given an org_domain or org_id. + if self._source: + return self._source + elif 'source' in self.data: + self._source = self.data['source'] + return self._source + + session = local_session(self.manager.session_factory) + + # Resolve Organization Id + if 'org-id' in self.data: + org_id = self.data['org-id'] + else: + orgs = session.client('cloudresourcemanager', 'v1', 'organizations') + res = orgs.execute_query( + 'search', {'body': { + 'filter': 'domain:%s' % self.data['org-domain']}}).get( + 'organizations') + if not res: + raise PolicyExecutionError("Could not determine organization id") + org_id = res[0]['name'].rsplit('/', 1)[-1] + + # Resolve Source + client = session.client(self.Service, self.ServiceVersion, 'organizations.sources') + source = None + res = [s for s in + client.execute_query( + 'list', {'parent': 'organizations/{}'.format(org_id)}).get('sources') + if s['displayName'] == self.CustodianSourceName] + if res: + source = res[0]['name'] + + if source is None: + source = client.execute_command( + 'create', + {'parent': 'organizations/{}'.format(org_id), + 'body': { + 'displayName': self.CustodianSourceName, + 'description': 'Cloud Management Rules Engine'}}).get('name') + self.log.info( + "policy:%s resolved cscc source: %s, update policy with this source value", + self.manager.ctx.policy.name, + source) + self._source = source + return self._source + + def get_name(self, r): + """Given an arbitrary resource attempt to resolve back to a qualified name.""" + namer = ResourceNameAdapters[self.manager.resource_type.service] + return namer(r) + + def get_finding(self, resource): + policy = self.manager.ctx.policy + resource_name = self.get_name(resource) + # ideally we could be using shake, but its py3.6+ only + finding_id = hashlib.sha256( + b"%s%s" % ( + policy.name.encode('utf8'), + resource_name.encode('utf8'))).hexdigest()[:32] + + finding = { + 'name': '{}/findings/{}'.format(self._source, finding_id), + 'resourceName': resource_name, + 'state': 'ACTIVE', + 'category': self.data.get('category', self.DefaultCategory), + 'eventTime': datetime.datetime.utcnow().isoformat('T') + 'Z', + 'sourceProperties': { + 'resource_type': self.manager.type, + 'title': policy.data.get('title', policy.name), + 'policy_name': policy.name, + 'policy': json.dumps(policy.data) + } + } + + request = { + 'parent': self._source, + 'findingId': finding_id[:31], + 'body': finding} + return request + + @classmethod + def register_resource(klass, registry, resource_class): + if resource_class.resource_type.service not in ResourceNameAdapters: + return + if 'post-finding' in resource_class.action_registry: + return + resource_class.action_registry.register('post-finding', klass) + + +# CSCC uses its own notion of resource id, if we want our findings on +# a resource to be linked from the asset view we need to post w/ the +# same resource name. If this conceptulization of resource name is +# standard, then we should move these to resource types with +# appropriate hierarchies by service. + + +def name_compute(r): + prefix = urlparse(r['selfLink']).path.strip('/').split('/')[2:][:-1] + return "//compute.googleapis.com/{}/{}".format( + "/".join(prefix), + r['id']) + + +def name_iam(r): + return "//iam.googleapis.com/projects/{}/serviceAccounts/{}".format( + r['projectId'], + r['uniqueId']) + + +def name_resourcemanager(r): + rid = r.get('projectNumber') + if rid is not None: + rtype = 'projects' + else: + rid = r.get('organizationId') + rtype = 'organizations' + return "//cloudresourcemanager.googleapis.com/{}/{}".format( + rtype, rid) + + +def name_container(r): + return "//container.googleapis.com/{}".format( + "/".join(urlparse(r['selfLink']).path.strip('/').split('/')[1:])) + + +def name_storage(r): + return "//storage.googleapis.com/{}".format(r['name']) + + +def name_appengine(r): + return "//appengine.googleapis.com/{}".format(r['name']) + + +ResourceNameAdapters = { + 'appengine': name_appengine, + 'cloudresourcemanager': name_resourcemanager, + 'compute': name_compute, + 'container': name_container, + 'iam': name_iam, + 'storage': name_storage, +} + +vsphere_resources.subscribe(PostFinding.register_resource) diff --git a/tools/c7n_vsphere/c7n_vsphere/client.py b/tools/c7n_vsphere/c7n_vsphere/client.py new file mode 100644 index 00000000000..90d7e7f921c --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/client.py @@ -0,0 +1,39 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2017 The Forseti Security Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import requests +from vmware.vapi.vsphere.client import create_vsphere_client + +session = requests.session() +session.verify = False +log = logging.getLogger('custodian.vsphere.client') + + +class Session: + def __init__(self, regionId=None): + self.username = os.getenv('VSPHERE_USERNAME') + self.password = os.getenv('VSPHERE_PASSWORD') + self.server = os.getenv('VSPHERE_ENDPOINT') + if not regionId: + regionId = os.getenv('VSPHERE_DEFAULT_REGION') + self.regionId = regionId + + def client(self): + vsphere_client = create_vsphere_client(server=os.getenv('VSPHERE_ENDPOINT'), username=os.getenv('VSPHERE_USERNAME'), password=os.getenv('VSPHERE_PASSWORD'), session=session) + return vsphere_client \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/entry.py b/tools/c7n_vsphere/c7n_vsphere/entry.py new file mode 100644 index 00000000000..8d9dda2359b --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/entry.py @@ -0,0 +1,17 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +import logging +from c7n_vsphere.resources import ( + vm, +) + +log = logging.getLogger('custodian.vsphere') + +ALL = [ + vm] + + +def initialize_vsphere(): + """vsphere entry point + """ \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/filters/__init__.py b/tools/c7n_vsphere/c7n_vsphere/filters/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/filters/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/filters/filter.py b/tools/c7n_vsphere/c7n_vsphere/filters/filter.py new file mode 100644 index 00000000000..a985a00ed36 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/filters/filter.py @@ -0,0 +1,340 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +import json + +import jmespath + +from c7n.exceptions import PolicyValidationError +from c7n.filters.core import Filter +from c7n.filters.core import ValueFilter + + +class PolicyFilter: + pass + +class Filter(Filter): + + def validate(self): + return self + + def __call__(self, i): + return i + +class vSphereFilter(Filter): + + def validate(self): + return self + + def __call__(self, i): + return self.get_request(i) + +class SGPermission(Filter): + """Filter for verifying security group ingress and egress permissions + + All attributes of a security group permission are available as + value filters. + + If multiple attributes are specified the permission must satisfy + all of them. Note that within an attribute match against a list value + of a permission we default to or. + + If a group has any permissions that match all conditions, then it + matches the filter. + + Permissions that match on the group are annotated onto the group and + can subsequently be used by the remove-permission action. + + We have specialized handling for matching `Ports` in ingress/egress + permission From/To range. The following example matches on ingress + rules which allow for a range that includes all of the given ports. + + .. code-block:: yaml + + - type: ingress + Ports: [22, 443, 80] + + As well for verifying that a rule only allows for a specific set of ports + as in the following example. The delta between this and the previous + example is that if the permission allows for any ports not specified here, + then the rule will match. ie. OnlyPorts is a negative assertion match, + it matches when a permission includes ports outside of the specified set. + + .. code-block:: yaml + + - type: ingress + OnlyPorts: [22] + + For simplifying ipranges handling which is specified as a list on a rule + we provide a `Cidr` key which can be used as a value type filter evaluated + against each of the rules. If any iprange cidr match then the permission + matches. + + .. code-block:: yaml + + - type: ingress + IpProtocol: -1 + FromPort: 445 + + We also have specialized handling for matching self-references in + ingress/egress permissions. The following example matches on ingress + rules which allow traffic its own same security group. + + .. code-block:: yaml + + - type: ingress + SelfReference: True + + As well for assertions that a ingress/egress permission only matches + a given set of ports, *note* OnlyPorts is an inverse match. + + .. code-block:: yaml + + - type: egress + OnlyPorts: [22, 443, 80] + + - type: egress + Cidr: + value_type: cidr + op: in + value: x.y.z + + `Cidr` can match ipv4 rules and `CidrV6` can match ipv6 rules. In + this example we are blocking global inbound connections to SSH or + RDP. + + .. code-block:: yaml + + - type: ingress + Ports: [22, 3389] + Cidr: + value: + - "0.0.0.0/0" + - "::/0" + op: in + + `SGReferences` can be used to filter out SG references in rules. + In this example we want to block ingress rules that reference a SG + that is tagged with `Access: Public`. + + .. code-block:: yaml + + - type: ingress + SGReferences: + key: "tag:Access" + value: "Public" + op: equal + + We can also filter SG references based on the VPC that they are + within. In this example we want to ensure that our outbound rules + that reference SGs are only referencing security groups within a + specified VPC. + + .. code-block:: yaml + + - type: egress + SGReferences: + key: 'VpcId' + value: 'vpc-11a1a1aa' + op: equal + + Likewise, we can also filter SG references by their description. + For example, we can prevent egress rules from referencing any + SGs that have a description of "default - DO NOT USE". + + .. code-block:: yaml + + - type: egress + SGReferences: + key: 'Description' + value: 'default - DO NOT USE' + op: equal + + """ + + perm_attrs = { + 'IpProtocol', "Priority", 'Policy'} + filter_attrs = { + 'Cidr', 'CidrV6', 'Ports', 'OnlyPorts', + 'SelfReference', 'Description', 'SGReferences'} + attrs = perm_attrs.union(filter_attrs) + attrs.add('match-operator') + attrs.add('match-operator') + + def validate(self): + delta = set(self.data.keys()).difference(self.attrs) + delta.remove('type') + if delta: + raise PolicyValidationError("Unknown keys %s on %s" % ( + ", ".join(delta), self.manager.data)) + return self + + def process(self, resources, event=None): + self.vfilters = [] + fattrs = list(sorted(self.perm_attrs.intersection(self.data.keys()))) + self.ports = 'Ports' in self.data and self.data['Ports'] or () + self.only_ports = ( + 'OnlyPorts' in self.data and self.data['OnlyPorts'] or ()) + for f in fattrs: + fv = self.data.get(f) + if isinstance(fv, dict): + fv['key'] = f + else: + fv = {f: fv} + vf = ValueFilter(fv, self.manager) + vf.annotate = False + + self.vfilters.append(vf) + + return super(SGPermission, self).process(resources, event) + + def process_ports(self, perm): + found = None + if perm['remote_ip_prefix'] == '0.0.0.0/0' or perm['remote_ip_prefix'] == '::/0': + return True + if perm['port_range_min'] is None or perm['port_range_max'] is None : + return True + FromPort = int(perm['port_range_min']) + ToPort = int(perm['port_range_max']) + for port in self.ports: + if port >= FromPort and port <= ToPort: + found = True + break + elif FromPort == -1 and ToPort == -1: + found = True + break + else: + found = False + only_found = False + for port in self.only_ports: + if port == FromPort and port == ToPort: + only_found = True + if self.only_ports and not only_found: + found = found is None or found and True or False + if self.only_ports and only_found: + found = False + return found + + + def _process_cidr(self, cidr_key, cidr_type, SourceCidrIp, perm): + found = None + SourceCidrIp = perm.get(SourceCidrIp, "") + if not SourceCidrIp: + return False + SourceCidrIp = {cidr_type: SourceCidrIp} + match_range = self.data[cidr_key] + if isinstance(match_range, dict): + match_range['key'] = cidr_type + else: + match_range = {cidr_type: match_range} + vf = ValueFilter(match_range, self.manager) + vf.annotate = False + found = vf(SourceCidrIp) + if found: + found = True + else: + found = False + return found + + def process_cidrs(self, perm, ipv4Cidr, ipv6Cidr): + found_v6 = found_v4 = None + if 'CidrV6' in self.data: + found_v6 = self._process_cidr('CidrV6', 'CidrIpv6', ipv6Cidr, perm) + if 'Cidr' in self.data: + found_v4 = self._process_cidr('Cidr', 'CidrIp', ipv4Cidr, perm) + match_op = self.data.get('match-operator', 'and') == 'and' and all or any + cidr_match = [k for k in (found_v6, found_v4) if k is not None] + if not cidr_match: + return None + return match_op(cidr_match) + + def process_description(self, perm): + if 'Description' not in self.data: + return None + + d = dict(self.data['Description']) + d['key'] = 'Description' + + vf = ValueFilter(d, self.manager) + vf.annotate = False + + for k in ('Ipv6Ranges', 'IpRanges', 'UserIdGroupPairs', 'PrefixListIds'): + if k not in perm or not perm[k]: + continue + return vf(perm[k][0]) + return False + + def process_self_reference(self, perm, sg_id): + found = None + ref_match = self.data.get('SelfReference') + if ref_match is not None: + found = False + if 'UserIdGroupPairs' in perm and 'SelfReference' in self.data: + self_reference = sg_id in [p['GroupId'] + for p in perm['UserIdGroupPairs']] + if ref_match is False and not self_reference: + found = True + if ref_match is True and self_reference: + found = True + return found + + def process_sg_references(self, perm, owner_id): + sg_refs = self.data.get('SGReferences') + if not sg_refs: + return None + + sg_perm = perm.get('UserIdGroupPairs', []) + if not sg_perm: + return False + + sg_group_ids = [p['GroupId'] for p in sg_perm if p['UserId'] == owner_id] + sg_resources = self.manager.get_resources(sg_group_ids) + vf = ValueFilter(sg_refs, self.manager) + vf.annotate = False + + for sg in sg_resources: + if vf(sg): + return True + return False + + def __call__(self, resource): + result = self.securityGroupAttributeRequst(resource) + matched = [] + match_op = self.data.get('match-operator', 'and') == 'and' and all or any + for perm in jmespath.search(self.ip_permissions_key, result): + perm_matches = {} + perm_matches['ports'] = self.process_ports(perm) + perm_matches['cidrs'] = self.process_self_cidrs(perm) + perm_match_values = list(filter( + lambda x: x is not None, perm_matches.values())) + # account for one python behavior any([]) == False, all([]) == True + if match_op == all and not perm_match_values: + continue + + match = match_op(perm_match_values) + if match: + matched.append(perm) + if matched: + return True + +SGPermissionSchema = { + 'match-operator': {'type': 'string', 'enum': ['or', 'and']}, + 'Ports': {'type': 'array', 'items': {'type': 'integer'}}, + 'OnlyPorts': {'type': 'array', 'items': {'type': 'integer'}}, + 'Policy': {}, + 'IpProtocol': { + 'oneOf': [ + {'enum': ["-1", -1, 'TCP', 'UDP', 'ICMP', 'ICMPV6']}, + {'$ref': '#/definitions/filters/value'} + ] + }, + 'FromPort': {'oneOf': [ + {'$ref': '#/definitions/filters/value'}, + {'type': 'integer'}]}, + 'ToPort': {'oneOf': [ + {'$ref': '#/definitions/filters/value'}, + {'type': 'integer'}]}, + 'IpRanges': {}, + 'Cidr': {}, + 'CidrV6': {}, + 'SGReferences': {} +} \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/filters/labels.py b/tools/c7n_vsphere/c7n_vsphere/filters/labels.py new file mode 100644 index 00000000000..e771c8032b8 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/filters/labels.py @@ -0,0 +1,119 @@ +# Copyright 2019 Karol Lassak +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta + +from c7n.filters import Filter, FilterValidationError +from c7n.filters.offhours import Time +from c7n.utils import type_schema + +DEFAULT_TAG = "custodian_status" + + +class LabelActionFilter(Filter): + """Filter resources for label specified future action + + Filters resources by a 'custodian_status' label which specifies a future + date for an action. + + The filter parses the label values looking for an 'op@date' + string. The date is parsed and compared to do today's date, the + filter succeeds if today's date is gte to the target date. + + The optional 'skew' parameter provides for incrementing today's + date a number of days into the future. An example use case might + be sending a final notice email a few days before terminating an + instance, or snapshotting a volume prior to deletion. + + The optional 'skew_hours' parameter provides for incrementing the current + time a number of hours into the future. + + Optionally, the 'tz' parameter can get used to specify the timezone + in which to interpret the clock (default value is 'utc') + + :example: + + .. code-block :: yaml + + policies: + - name: vm-stop-marked + resource: vsphere.instance + filters: + - type: marked-for-op + # The default label used is custodian_status + # but that is configurable + label: custodian_status + op: stop + # Another optional label is skew + tz: utc + + + """ + schema = type_schema( + 'marked-for-op', + label={'type': 'string'}, + tz={'type': 'string'}, + skew={'type': 'number', 'minimum': 0}, + skew_hours={'type': 'number', 'minimum': 0}, + op={'type': 'string'}) + + def validate(self): + op = self.data.get('op') + if self.manager and op not in self.manager.action_registry.keys(): + raise FilterValidationError( + "Invalid marked-for-op op:%s in %s" % (op, self.manager.data)) + + tz = Time.get_tz(self.data.get('tz', 'utc')) + if not tz: + raise FilterValidationError( + "Invalid timezone specified '%s' in %s" % ( + self.data.get('tz'), self.manager.data)) + return self + + def process(self, resources, event=None): + self.label = self.data.get('label', DEFAULT_TAG) + self.op = self.data.get('op', 'stop') + self.skew = self.data.get('skew', 0) + self.skew_hours = self.data.get('skew_hours', 0) + self.tz = Time.get_tz(self.data.get('tz', 'utc')) + return super(LabelActionFilter, self).process(resources, event) + + def __call__(self, i): + v = i.get('labels', {}).get(self.label, None) + + if v is None: + return False + if '-' not in v or '_' not in v: + return False + + msg, action, action_date_str = v.rsplit('-', 2) + + if action != self.op: + return False + + try: + action_date = datetime.strptime(action_date_str, '%Y_%m_%d__%H_%M') + except Exception: + self.log.error("could not parse label:%s value:%s on %s" % ( + self.label, v, i['name'])) + return False + + # current_date must match timezones with the parsed date string + if action_date.tzinfo: + action_date = action_date.astimezone(self.tz) + current_date = datetime.now(tz=self.tz) + else: + current_date = datetime.now() + + return current_date >= (action_date - timedelta(days=self.skew, hours=self.skew_hours)) diff --git a/tools/c7n_vsphere/c7n_vsphere/handler.py b/tools/c7n_vsphere/c7n_vsphere/handler.py new file mode 100644 index 00000000000..f2d7bbce95b --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/handler.py @@ -0,0 +1,56 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import uuid + +# Load resource plugins +from c7n_vsphere.entry import initialize_vsphere + +from c7n.config import Config +from c7n.loader import PolicyLoader + +initialize_vsphere() + +log = logging.getLogger('custodian.vsphere.functions') + +logging.getLogger().setLevel(logging.INFO) + + +def run(event, context=None): + # policies file should always be valid in functions so do loading naively + with open('config.json') as f: + policy_config = json.load(f) + + if not policy_config or not policy_config.get('policies'): + log.error('Invalid policy config') + return False + + # setup execution options + options = Config.empty(**policy_config.pop('execution-options', {})) + options.update( + policy_config['policies'][0].get('mode', {}).get('execution-options', {})) + # if output_dir specified use that, otherwise make a temp directory + if not options.output_dir: + options['output_dir'] = get_tmp_output_dir() + + loader = PolicyLoader(options) + policies = loader.load_data(policy_config, 'config.json', validate=False) + if policies: + for p in policies: + log.info("running policy %s", p.name) + p.validate() + p.push(event, context) + return True + + +def get_tmp_output_dir(): + output_dir = '/tmp/' + str(uuid.uuid4()) + if not os.path.exists(output_dir): + try: + os.mkdir(output_dir) + except OSError as error: + log.warning("Unable to make output directory: {}".format(error)) + return output_dir diff --git a/tools/c7n_vsphere/c7n_vsphere/mu.py b/tools/c7n_vsphere/c7n_vsphere/mu.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/mu.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/output.py b/tools/c7n_vsphere/c7n_vsphere/output.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/output.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/policy.py b/tools/c7n_vsphere/c7n_vsphere/policy.py new file mode 100644 index 00000000000..d559235032d --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/policy.py @@ -0,0 +1,182 @@ +# Copyright 2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from c7n_vsphere import mu +from dateutil.tz import tz + +from c7n.exceptions import PolicyValidationError +from c7n.policy import execution, ServerlessExecutionMode, PullMode +from c7n.utils import local_session, type_schema + +DEFAULT_REGION = 'us-central1' + + +class FunctionMode(ServerlessExecutionMode): + + schema = type_schema( + 'vsphere', + **{'execution-options': {'$ref': '#/definitions/basic_dict'}, + 'timeout': {'type': 'string'}, + 'memory-size': {'type': 'integer'}, + 'labels': {'$ref': '#/definitions/string_dict'}, + 'network': {'type': 'string'}, + 'max-instances': {'type': 'integer'}, + 'service-account': {'type': 'string'}, + 'environment': {'$ref': '#/definitions/string_dict'}} + ) + + def __init__(self, policy): + self.policy = policy + self.log = logging.getLogger('custodian.vsphere.funcexec') + self.region = policy.options.regions[0] if len(policy.options.regions) else DEFAULT_REGION + + def run(self): + raise NotImplementedError("subclass responsibility") + + def provision(self): + self.log.info("Provisioning policy function %s", self.policy.name) + manager = mu.CloudFunctionManager(self.policy.session_factory, self.region) + return manager.publish(self._get_function()) + + def deprovision(self): + manager = mu.CloudFunctionManager(self.policy.session_factory, self.region) + return manager.remove(self._get_function()) + + def validate(self): + pass + + def _get_function(self): + raise NotImplementedError("subclass responsibility") + + +@execution.register('vsphere-periodic') +class PeriodicMode(FunctionMode, PullMode): + """Deploy a policy as a Cloud Functions triggered by Cloud Scheduler + at user defined cron interval via Pub/Sub. + + Default region the function is deployed to is ``us-central1``. In + case you want to change that, use the cli ``--region`` flag. + """ + + schema = type_schema( + 'vsphere-periodic', + rinherit=FunctionMode.schema, + required=['schedule'], + **{'trigger-type': {'enum': ['http', 'pubsub']}, + 'tz': {'type': 'string'}, + 'schedule': {'type': 'string'}}) + + def validate(self): + mode = self.policy.data['mode'] + if 'tz' in mode: + error = PolicyValidationError( + "policy:%s vsphere-periodic invalid tz:%s" % ( + self.policy.name, mode['tz'])) + # We can't catch all errors statically, our local tz retrieval + # then the form vsphere is using, ie. not all the same aliases are + # defined. + tzinfo = tz.gettz(mode['tz']) + if tzinfo is None: + raise error + + def _get_function(self): + events = [mu.PeriodicEvent( + local_session(self.policy.session_factory), + self.policy.data['mode'], + self.region + )] + return mu.PolicyFunction(self.policy, events=events) + + def run(self, event, context): + return PullMode.run(self) + + +@execution.register('vsphere-audit') +class ApiAuditMode(FunctionMode): + """Custodian policy execution on vsphere api audit logs events. + + Deploys as a Cloud Function triggered by api calls. This allows + you to apply your policies as soon as an api call occurs. Audit + logs creates an event for every api call that occurs in your vsphere + account. See `vsphere Audit Logs + `_ for more + details. + + Default region the function is deployed to is + ``us-central1``. In case you want to change that, use the cli + ``--region`` flag. + """ + + schema = type_schema( + 'vsphere-audit', + methods={'type': 'array', 'items': {'type': 'string'}}, + required=['methods'], + rinherit=FunctionMode.schema) + + def resolve_resources(self, event): + """Resolve a vsphere resource from its audit trail metadata. + """ + if self.policy.resource_manager.resource_type.get_requires_event: + return [self.policy.resource_manager.get_resource(event)] + resource_info = event.get('resource') + if resource_info is None or 'labels' not in resource_info: + self.policy.log.warning("Could not find resource information in event") + return + # copy resource name, the api doesn't like resource ids, just names. + if 'resourceName' in event['protoPayload']: + resource_info['labels']['resourceName'] = event['protoPayload']['resourceName'] + + resource = self.policy.resource_manager.get_resource(resource_info['labels']) + return [resource] + + def _get_function(self): + events = [mu.ApiSubscriber( + local_session(self.policy.session_factory), + self.policy.data['mode'])] + return mu.PolicyFunction(self.policy, events=events) + + def validate(self): + if not self.policy.resource_manager.resource_type.get: + raise PolicyValidationError( + "Resource:%s does not implement retrieval method" % ( + self.policy.resource_type)) + + def run(self, event, context): + """Execute a vsphere serverless model""" + from c7n.actions import EventAction + + resources = self.resolve_resources(event) + if not resources: + return + + resources = self.policy.resource_manager.filter_resources( + resources, event) + + self.policy.log.info("Filtered resources %d" % len(resources)) + + if not resources: + return + + self.policy.ctx.metrics.put_metric( + 'ResourceCount', len(resources), 'Count', Scope="Policy", + buffer=False) + + for action in self.policy.resource_manager.actions: + if isinstance(action, EventAction): + action.process(resources, event) + else: + action.process(resources) + + return resources diff --git a/tools/c7n_vsphere/c7n_vsphere/provider.py b/tools/c7n_vsphere/c7n_vsphere/provider.py new file mode 100644 index 00000000000..3d686a0a1da --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/provider.py @@ -0,0 +1,34 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 + +from c7n.registry import PluginRegistry +from c7n.provider import Provider, clouds + +from .resources.resource_map import ResourceMap +from .client import Session + +import logging + +log = logging.getLogger('custodian.vsphere') + + +@clouds.register('vsphere') +class vSphere(Provider): + + display_name = 'vsphere' + resource_prefix = 'vsphere' + resources = PluginRegistry('%s.resources' % resource_prefix) + resource_map = ResourceMap + + def initialize(self, options): + return options + + def initialize_policies(self, policy_collection, options): + return policy_collection + + def get_session_factory(self, options): + """Get a credential/session factory for api usage.""" + return Session + + +resources = vSphere.resources \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/query.py b/tools/c7n_vsphere/c7n_vsphere/query.py new file mode 100644 index 00000000000..367427ff2f3 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/query.py @@ -0,0 +1,360 @@ +# Copyright 2017-2018 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import json +import logging + +import jmespath + +from c7n.actions import ActionRegistry +from c7n.filters import FilterRegistry +from c7n.manager import ResourceManager +from c7n.query import sources, MaxResourceLimit +from c7n.utils import local_session, chunks + +log = logging.getLogger('c7n_vsphere.query') + + +class ResourceQuery: + def __init__(self, session_factory): + self.session_factory = session_factory + + def filter(self, resource_manager, **params): + result = resource_manager.get_request() + global false, null, true + false = False + null = None + true = True + return eval(result) + + def _invoke_client_enum(self, client, request, params, path): + result = client.do_action_with_exception(request) + return jmespath.search(path, eval(result)) + +@sources.register('describe') +class DescribeSource: + + def __init__(self, manager): + self.manager = manager + self.query = ResourceQuery(manager.session_factory) + + + def get_resources(self, query): + if query is None: + query = {} + return self.query.filter(self.manager, **query) + + def get_permissions(self): + m = self.manager.resource_type + if m.permissions: + return m.permissions + method = m.enum_spec[0] + if method == 'aggregatedList': + method = 'list' + component = m.component + if '.' in component: + component = component.split('.')[-1] + return ("%s.%s.%s" % ( + m.perm_service or m.service, component, method),) + + def augment(self, resources): + return resources + + +@sources.register('inventory') +class AssetInventory: + + permissions = ("cloudasset.assets.searchAllResources", + "cloudasset.assets.exportResource") + + def __init__(self, manager): + self.manager = manager + + def get_resources(self, query): + session = local_session(self.manager.session_factory) + if query is None: + query = {} + if 'scope' not in query: + query['scope'] = 'projects/%s' % session.get_default_project() + if 'assetTypes' not in query: + query['assetTypes'] = [self.manager.resource_type.asset_type] + + search_client = session.client('cloudasset', 'v1p1beta1', 'resources') + resource_client = session.client('cloudasset', 'v1', 'v1') + resources = [] + + results = list(search_client.execute_paged_query('searchAll', query)) + for resource_set in chunks(itertools.chain(*[rs['results'] for rs in results]), 100): + rquery = { + 'parent': query['scope'], + 'contentType': 'RESOURCE', + 'assetNames': [r['name'] for r in resource_set]} + for history_result in resource_client.execute_query( + 'batchGetAssetsHistory', rquery).get('assets', ()): + resource = history_result['asset']['resource']['data'] + resource['c7n:history'] = { + 'window': history_result['window'], + 'ancestors': history_result['asset']['ancestors']} + resources.append(resource) + return resources + + def get_permissions(self): + return self.permissions + + def augment(self, resources): + return resources + + +class QueryMeta(type): + """metaclass to have consistent action/filter registry for new resources.""" + def __new__(cls, name, parents, attrs): + if 'filter_registry' not in attrs: + attrs['filter_registry'] = FilterRegistry( + '%s.filters' % name.lower()) + if 'action_registry' not in attrs: + attrs['action_registry'] = ActionRegistry( + '%s.actions' % name.lower()) + + return super(QueryMeta, cls).__new__(cls, name, parents, attrs) + + +class QueryResourceManager(ResourceManager, metaclass=QueryMeta): + + def __init__(self, data, options): + super(QueryResourceManager, self).__init__(data, options) + self.source = self.get_source(self.source_type) + + + def get_permissions(self): + return self.source.get_permissions() + + def get_source(self, source_type): + return sources.get(source_type)(self) + + def get_client(self): + return local_session(self.session_factory).client( + self.resource_type.service, + self.resource_type.version, + self.resource_type.component) + + def get_model(self): + return self.resource_type + + def get_cache_key(self, query): + return {'source_type': self.source_type, + 'query': query, + 'service': self.resource_type.service, + 'version': self.resource_type.version, + 'component': self.resource_type.component} + + def get_resource(self, resource_info): + + return self.resource_type.get(self.get_client(), resource_info) + + @property + def source_type(self): + return self.data.get('source', 'describe') + + def get_resource_query(self): + if 'query' in self.data: + return {'filter': self.data.get('query')} + + def resources(self, query=None): + q = query or self.get_resource_query() + key = self.get_cache_key(q) + resources = self._fetch_resources(q) + self._cache.save(key, resources) + + resource_count = len(resources) + resources = self.filter_resources(resources) + # Check if we're out of a policies execution limits. + + if self.data == self.ctx.policy.data: + self.check_resource_limit(len(resources), resource_count) + + return resources + + def check_resource_limit(self, selection_count, population_count): + """Check if policy's execution affects more resources then its limit. + """ + p = self.ctx.policy + max_resource_limits = MaxResourceLimit(p, selection_count, population_count) + return max_resource_limits.check_resource_limits() + + def _fetch_resources(self, query): + try: + return self.augment(self.source.get_resources(query)) or [] + except Exception as e: + error = extract_error(e) + if error is None: + raise + elif error == 'accessNotConfigured': + log.warning( + "Resource:%s not available -> Service:%s not enabled on %s", + self.type, + self.resource_type.service, + local_session(self.session_factory).get_default_project()) + return [] + raise + + def augment(self, resources): + return resources + + +class ChildResourceManager(QueryResourceManager): + + def get_resource(self, resource_info): + child_instance = super(ChildResourceManager, self).get_resource(resource_info) + + parent_resource = self.resource_type.parent_spec['resource'] + parent_instance = self.get_resource_manager(parent_resource).get_resource( + self._get_parent_resource_info(child_instance) + ) + + annotation_key = self.resource_type.get_parent_annotation_key() + child_instance[annotation_key] = parent_instance + + return child_instance + + def _fetch_resources(self, query): + if not query: + query = {} + + resources = [] + annotation_key = self.resource_type.get_parent_annotation_key() + parent_query = self.get_parent_resource_query() + parent_resource_manager = self.get_resource_manager( + resource_type=self.resource_type.parent_spec['resource'], + data=({'query': parent_query} if parent_query else {}) + ) + + for parent_instance in parent_resource_manager.resources(): + query.update(self._get_child_enum_args(parent_instance)) + children = super(ChildResourceManager, self)._fetch_resources(query) + + for child_instance in children: + child_instance[annotation_key] = parent_instance + + resources.extend(children) + + return resources + + def _get_parent_resource_info(self, child_instance): + mappings = self.resource_type.parent_spec['parent_get_params'] + return self._extract_fields(child_instance, mappings) + + def _get_child_enum_args(self, parent_instance): + mappings = self.resource_type.parent_spec['child_enum_params'] + return self._extract_fields(parent_instance, mappings) + + def get_parent_resource_query(self): + parent_spec = self.resource_type.parent_spec + enabled = parent_spec['use_child_query'] if 'use_child_query' in parent_spec else False + if enabled and 'query' in self.data: + return self.data.get('query') + + @staticmethod + def _extract_fields(source, mappings): + result = {} + + for mapping in mappings: + result[mapping[1]] = jmespath.search(mapping[0], source) + + return result + + +class TypeMeta(type): + + def __repr__(cls): + return "" % ( + cls.service) + + +class TypeInfo(metaclass=TypeMeta): + + # api client construction information + service = None + version = None + component = None + + +class ChildTypeInfo(TypeInfo): + + parent_spec = None + + @classmethod + def get_parent_annotation_key(cls): + parent_resource = cls.parent_spec['resource'] + return 'c7n:{}'.format(parent_resource) + + +ERROR_REASON = jmespath.compile('error.errors[0].reason') + + +def extract_error(e): + + try: + edata = json.loads(e.content) + except Exception: + return None + return ERROR_REASON.search(edata) + + +class AliyunLocation: + """ + The `_locations` dict is formed by the string keys representing locations taken from + `KMS `_ and + `App Engine `_ and list values containing the string names of the services + the locations are available for. + """ + _locations = {'eur4': ['kms'], + 'global': ['kms'], + 'europe-west4': ['kms'], + 'asia-east2': ['appengine', 'kms'], + 'asia-east1': ['kms'], + 'asia': ['kms'], + 'europe-north1': ['kms'], + 'us-central1': ['kms'], + 'nam4': ['kms'], + 'asia-southeast1': ['kms'], + 'europe': ['kms'], + 'australia-southeast1': ['appengine', 'kms'], + 'us-central': ['appengine'], + 'asia-south1': ['appengine', 'kms'], + 'us-west1': ['kms'], + 'us-west2': ['appengine', 'kms'], + 'asia-northeast2': ['appengine', 'kms'], + 'asia-northeast1': ['appengine', 'kms'], + 'europe-west2': ['appengine', 'kms'], + 'europe-west3': ['appengine', 'kms'], + 'us-east4': ['appengine', 'kms'], + 'europe-west1': ['kms'], + 'europe-west6': ['appengine', 'kms'], + 'us': ['kms'], + 'us-east1': ['appengine', 'kms'], + 'northamerica-northeast1': ['appengine', 'kms'], + 'europe-west': ['appengine'], + 'southamerica-east1': ['appengine', 'kms']} + + @classmethod + def get_service_locations(cls, service): + """ + Returns a list of the locations that have a given service in associated value lists. + + :param service: a string representing the name of a service locations are queried for + """ + return [location for location in AliyunLocation._locations + if service in AliyunLocation._locations[location]] diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/__init__.py b/tools/c7n_vsphere/c7n_vsphere/resources/__init__.py new file mode 100644 index 00000000000..cdce4e86dfd --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/__init__.py @@ -0,0 +1,2 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/cluster.py b/tools/c7n_vsphere/c7n_vsphere/resources/cluster.py new file mode 100644 index 00000000000..66b2d6589fc --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/cluster.py @@ -0,0 +1,67 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('cluster') +class Cluster(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'cluster' + name = 'name' + default_report_fields = ['cluster', 'name'] + + def get_request(self): + client = Session.client(self) + clusters = client.vcenter.Cluster.list() + #Summary(cluster='domain-c33', name='cluster-try', ha_enabled=False, drs_enabled=False) + res = [] + for item in clusters: + cluster = client.vcenter.Cluster.get(item.cluster) + data= { + "F2CId": item.cluster, + "cluster": item.cluster, + "name": item.name, + "ha_enabled": item.ha_enabled, + "drs_enabled": item.drs_enabled, + "resource_pool": cluster.resource_pool + } + res.append(data) + return json.dumps(res) + +@Cluster.filter_registry.register('ha-enabled') +class HaFilter(Filter): + """Filters Clusters based on their ha_enabled + :example: + .. code-block:: yaml + policies: + - name: vsphere-cluster-ha-enabled + resource: vsphere.cluster + filters: + - not: + - type: ha-enabled + value: true + """ + schema = type_schema( + 'ha-enabled', + value={'type': 'boolean'}, + ) + + def process(self, resources, event=None): + results = [] + value = self.data.get('value', None) + for cluster in resources: + matched = True + if value is not None and value != cluster.get('ha_enabled'): + matched = False + if matched: + results.append(cluster) + return results + diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/datacenter.py b/tools/c7n_vsphere/c7n_vsphere/resources/datacenter.py new file mode 100644 index 00000000000..325eeebcbc6 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/datacenter.py @@ -0,0 +1,78 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('datacenter') +class Datacenter(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'datacenter' + name = 'name' + default_report_fields = ['datacenter', 'name'] + + def get_request(self): + client = Session.client(self) + datacenters = client.vcenter.Datacenter.list() + #Summary(datacenter='datacenter-2', name='Datacenter') + res = [] + for item in datacenters: + #{name : Datacenter, datastore_folder : group-s5, host_folder : group-h4, network_folder : group-n6, vm_folder : group-v3} + datacenter = client.vcenter.Datacenter.get(item.datacenter) + data= { + "F2CId": item.datacenter, + "datacenter": item.datacenter, + "name": item.name, + "datastore_folder": datacenter.datastore_folder, + "host_folder": datacenter.host_folder, + "network_folder": datacenter.network_folder, + "vm_folder": datacenter.vm_folder, + } + res.append(data) + return json.dumps(res) + +@Datacenter.filter_registry.register('system') +class SystemFilter(Filter): + """Filters Datacenters based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-datacenter-system + resource: vsphere.datacenter + filters: + - not: + - type: system + host_folder: '' + network_folder: '' + vm_folder: '' + """ + schema = type_schema( + 'system', + host_folder={'type': 'string'}, + network_folder={'type': 'string'}, + vm_folder={'type': 'string'}, + ) + + def process(self, resources, event=None): + results = [] + host_folder = self.data.get('host_folder', None) + network_folder = self.data.get('network_folder', None) + vm_folder = self.data.get('vm_folder', None) + for datacenter in resources: + matched = True + if host_folder == datacenter.get('host_folder'): + matched = False + if network_folder == datacenter.get('network_folder'): + matched = False + if vm_folder == datacenter.get('vm_folder'): + matched = False + if matched: + results.append(datacenter) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/datastore.py b/tools/c7n_vsphere/c7n_vsphere/resources/datastore.py new file mode 100644 index 00000000000..871860e6637 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/datastore.py @@ -0,0 +1,80 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('datastore') +class Datastore(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'datastore' + name = 'name' + default_report_fields = ['datastore', 'name'] + + def get_request(self): + client = Session.client(self) + datastores = client.vcenter.Datastore.list() + #Summary(datastore='datastore-1013', name='datastore1', type=Type(string='VMFS'), free_space=610890940416, capacity=991600574464) + res = [] + for item in datastores: + #{name : datastore1, type : VMFS, accessible : True, free_space : 610536521728, multiple_host_access : False, thin_provisioning_supported : True} + datastore = client.vcenter.Datastore.get(item.datastore) + data= { + "F2CId": item.datastore, + "datastore": item.datastore, + "name": item.name, + "type": str(item.type), + "free_space": item.free_space, + "capacity": item.capacity, + "accessible": datastore.accessible, + "multiple_host_access": datastore.multiple_host_access, + "thin_provisioning_supported": datastore.thin_provisioning_supported, + } + res.append(data) + return json.dumps(res) + +@Datastore.filter_registry.register('system') +class SystemFilter(Filter): + """Filters Datastores based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-datastore-system + resource: vsphere.datastore + filters: + - not: + - type: system + free_space: 500 + thin_provisioning_supported: true + multiple_host_access: true + """ + schema = type_schema( + 'system', + free_space={'type': 'number'}, + thin_provisioning_supported={'type': 'boolean'}, + multiple_host_access={'type': 'boolean'}, + ) + + def process(self, resources, event=None): + results = [] + free_space = self.data.get('free_space', None) + thin_provisioning_supported = self.data.get('thin_provisioning_supported', None) + multiple_host_access = self.data.get('multiple_host_access', None) + for datastore in resources: + matched = True + if free_space is not None and free_space*1024*1024*1024 > datastore.get('free_space'): + matched = False + if thin_provisioning_supported and thin_provisioning_supported != datastore.get('thin_provisioning_supported'): + matched = False + if multiple_host_access and multiple_host_access != datastore.get('multiple_host_access'): + matched = False + if matched: + results.append(datastore) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/folder.py b/tools/c7n_vsphere/c7n_vsphere/resources/folder.py new file mode 100644 index 00000000000..004b5850164 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/folder.py @@ -0,0 +1,31 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n_vsphere.query import QueryResourceManager, TypeInfo +from c7n_vsphere.provider import resources +from c7n_vsphere.client import Session + +@resources.register('folder') +class Folder(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'folder' + name = 'name' + default_report_fields = ['folder', 'name'] + + def get_request(self): + client = Session.client(self) + folders = client.vcenter.Folder.list() + #Summary(folder='group-d1', name='Datacenters', type=Type(string='DATACENTER')) + res = [] + for item in folders: + data= { + "F2CId": item.folder, + "folder": item.folder, + "name": item.name, + "type": str(item.type), + } + res.append(data) + return json.dumps(res) \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/host.py b/tools/c7n_vsphere/c7n_vsphere/resources/host.py new file mode 100644 index 00000000000..d9ecc1d02ab --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/host.py @@ -0,0 +1,69 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('host') +class Host(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'host' + name = 'name' + default_report_fields = ['host', 'name'] + + def get_request(self): + client = Session.client(self) + hosts = client.vcenter.Host.list() + #Summary(host='host-10', name='10.1.240.15', connection_state=ConnectionState(string='CONNECTED'), power_state=PowerState(string='POWERED_ON')) + res = [] + for item in hosts: + data= { + "F2CId": item.host, + "host": item.host, + "name": item.name, + "connection_state": str(item.connection_state), + "power_state": str(item.power_state), + } + res.append(data) + return json.dumps(res) + +@Host.filter_registry.register('system') +class SystemFilter(Filter): + """Filters Hosts based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-host-system + resource: vsphere.host + filters: + - not: + - type: system + connection_state: CONNECTED + power_state: POWERED_ON + """ + schema = type_schema( + 'system', + connection_state={'type': 'string'}, + power_state={'type': 'string'}, + ) + + def process(self, resources, event=None): + results = [] + connection_state = self.data.get('connection_state', None) + power_state = self.data.get('power_state', None) + for host in resources: + matched = True + if connection_state is not None and connection_state != host.get('connection_state'): + matched = False + if power_state is not None and power_state != host.get('power_state'): + matched = False + if matched: + results.append(host) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/network.py b/tools/c7n_vsphere/c7n_vsphere/resources/network.py new file mode 100644 index 00000000000..c0273307b38 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/network.py @@ -0,0 +1,63 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('network') +class Network(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'network' + name = 'name' + default_report_fields = ['network', 'name'] + + def get_request(self): + client = Session.client(self) + networks = client.vcenter.Network.list() + #Summary(network='dvportgroup-1312', name='cluster-try-VSAN-DPortGroup', type=Type(string='DISTRIBUTED_PORTGROUP')) + res = [] + for item in networks: + data= { + "F2CId": item.network, + "network": item.network, + "name": item.name, + "type": str(item.type), + } + res.append(data) + return json.dumps(res) + +@Network.filter_registry.register('system') +class SystemFilter(Filter): + """Filters Networks based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-network-system + resource: vsphere.network + filters: + - not: + - type: system + value: DISTRIBUTED_PORTGROUP + """ + schema = type_schema( + 'system', + value={'type': 'string'}, + ) + + def process(self, resources, event=None): + results = [] + value = self.data.get('value', None) + for network in resources: + matched = True + if value is not None and value != network.get('type'): + matched = False + if matched: + results.append(network) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/resource_map.py b/tools/c7n_vsphere/c7n_vsphere/resources/resource_map.py new file mode 100644 index 00000000000..b92d3f20ec5 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/resource_map.py @@ -0,0 +1,12 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +ResourceMap = { + "vsphere.vm": "c7n_vsphere.resources.vm.VM", + "vsphere.cluster": "c7n_vsphere.resources.cluster.Cluster", + "vsphere.datacenter": "c7n_vsphere.resources.datacenter.Datacenter", + "vsphere.datastore": "c7n_vsphere.resources.datastore.Datastore", + "vsphere.folder": "c7n_vsphere.resources.folder.Folder", + "vsphere.host": "c7n_vsphere.resources.host.Host", + "vsphere.network": "c7n_vsphere.resources.network.Network", + "vsphere.resourcepool": "c7n_vsphere.resources.resourcepool.ResourcePool", +} \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/resourcepool.py b/tools/c7n_vsphere/c7n_vsphere/resources/resourcepool.py new file mode 100644 index 00000000000..9c562fefab5 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/resourcepool.py @@ -0,0 +1,74 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + + +@resources.register('resourcepool') +class ResourcePool(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'resource_pool' + name = 'name' + default_report_fields = ['resource_pool', 'name'] + + def get_request(self): + client = Session.client(self) + resource_pools = client.vcenter.ResourcePool.list() + #Summary(resource_pool='resgroup-34', name='Resources') + res = [] + for item in resource_pools: + #{name : Resources, resource_pools : set(), cpu_allocation : None, memory_allocation : None} + try: + resource_pool = client.vcenter.ResourcePool.get(item.resource_pool) + except: + resource_pool = None + data= { + "F2CId": item.resource_pool, + "resource_pool": item.resource_pool, + "name": item.name, + "cpu_allocation": str(resource_pool.cpu_allocation) if (resource_pool is not None) else None, + "memory_allocation": str(resource_pool.memory_allocation) if (resource_pool is not None) else None, + } + res.append(data) + return json.dumps(res) + +@ResourcePool.filter_registry.register('system') +class SystemFilter(Filter): + """Filters ResourcePools based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-resourcepool-system + resource: vsphere.resourcepool + filters: + - not: + - type: system + cpu_allocation: 0 + memory_allocation: 0 + """ + schema = type_schema( + 'system', + cpu_allocation={'type': 'number'}, + memory_allocation={'type': 'number'}, + ) + + def process(self, resources, event=None): + results = [] + cpu_allocation = self.data.get('cpu_allocation', None) + memory_allocation = self.data.get('memory_allocation', None) + for resourcepool in resources: + matched = True + if cpu_allocation is not None and cpu_allocation != resourcepool.get('cpu_allocation'): + matched = False + if memory_allocation is not None and memory_allocation != resourcepool.get('memory_allocation'): + matched = False + if matched: + results.append(resourcepool) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/c7n_vsphere/resources/vm.py b/tools/c7n_vsphere/c7n_vsphere/resources/vm.py new file mode 100644 index 00000000000..7e015ae3297 --- /dev/null +++ b/tools/c7n_vsphere/c7n_vsphere/resources/vm.py @@ -0,0 +1,91 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +# +import json +import logging + +from c7n.filters import Filter +from c7n.utils import type_schema +from c7n_vsphere.client import Session +from c7n_vsphere.provider import resources +from c7n_vsphere.query import QueryResourceManager, TypeInfo + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S') + +@resources.register('vm') +class VM(QueryResourceManager): + class resource_type(TypeInfo): + enum_spec = ('list', None) + id = 'vm' + name = 'name' + default_report_fields = ['vm', 'name'] + + def get_request(self): + client = Session.client(self) + vms = client.vcenter.VM.list() + #Summary(vm='vm-17', name='QA-PROXY', power_state=State(string='POWERED_ON'), cpu_count=2, memory_size_mib=4096) + #{guest_os : CENTOS_7_64, name : FIT2CLOUD-2.0-TRY2, identity : None, power_state : POWERED_ON, instant_clone_frozen : None, hardware : {version : VMX_14, upgrade_policy : NEVER, upgrade_version : None, upgrade_status : NONE, upgrade_error : None}, boot : {type : BIOS, efi_legacy_boot : None, network_protocol : None, delay : 0, retry : False, retry_delay : 10, enter_setup_mode : False}, boot_devices : [], cpu : {count : 4, cores_per_socket : 1, hot_add_enabled : False, hot_remove_enabled : False}, memory : {size_mib : 16384, hot_add_enabled : False, hot_add_increment_size_mib : None, hot_add_limit_mib : None}, disks : {'2000': Info(label='Hard disk 1', type=HostBusAdapterType(string='SCSI'), ide=None, scsi=ScsiAddressInfo(bus=0, unit=0), sata=None, backing=BackingInfo(type=BackingType(string='VMDK_FILE'), vmdk_file='[Local] FIT2CLOUD-2.0-TRY2/FIT2CLOUD-2.0-TRY2-000001.vmdk'), capacity=107374182400)}, nics : {'4000': Info(label='Network adapter 1', type=EmulationType(string='VMXNET3'), upt_compatibility_enabled=True, mac_type=MacAddressType(string='GENERATED'), mac_address='00:0c:29:24:5d:a3', pci_slot_number=192, wake_on_lan_enabled=False, backing=BackingInfo(type=BackingType(string='STANDARD_PORTGROUP'), network='network-12', network_name='VM Network', host_device=None, distributed_switch_uuid=None, distributed_port=None, connection_cookie=None, opaque_network_type=None, opaque_network_id=None), state=ConnectionState(string='CONNECTED'), start_connected=True, allow_guest_control=True)}, cdroms : {'16000': Info(type=HostBusAdapterType(string='SATA'), label='CD/DVD drive 1', ide=None, sata=SataAddressInfo(bus=0, unit=0), backing=BackingInfo(type=BackingType(string='ISO_FILE'), iso_file='[Local] iso/CentOS-7-x86_64-Minimal-1804.iso', host_device=None, auto_detect=None, device_access_type=None), state=ConnectionState(string='NOT_CONNECTED'), start_connected=True, allow_guest_control=True)}, floppies : {}, parallel_ports : {}, serial_ports : {}, sata_adapters : {'15000': Info(label='SATA controller 0', type=Type(string='AHCI'), bus=0, pci_slot_number=35)}, scsi_adapters : {'1000': Info(label='SCSI controller 0', type=Type(string='PVSCSI'), scsi=ScsiAddressInfo(bus=0, unit=7), pci_slot_number=160, sharing=Sharing(string='NONE'))}} + res = [] + for item in vms: + try: + vm = client.vcenter.VM.get(item.vm) + except: + vm = None + data= { + "F2CId": item.vm, + "vm": item.vm, + "name": item.name, + "power_state": str(item.power_state), + "cpu_count": item.cpu_count, + "memory_size_mib": item.memory_size_mib, + "guest_os": str(vm.guest_os) if (vm is not None) else None, + "identity": str(vm.identity) if (vm is not None) else None, + "instant_clone_frozen": vm.instant_clone_frozen if (vm is not None) else None, + "hardware": str(vm.hardware) if (vm is not None) else None, + "boot": str(vm.boot) if (vm is not None) else None, + "boot_devices": vm.boot_devices if (vm is not None) else None, + "cpu": str(vm.cpu) if (vm is not None) else None, + "memory": str(vm.memory) if (vm is not None) else None, + "disks": str(vm.disks) if (vm is not None) else None, + } + res.append(data) + return json.dumps(res) + +@VM.filter_registry.register('system') +class SystemFilter(Filter): + """Filters VMs based on their system + :example: + .. code-block:: yaml + policies: + - name: vsphere-vm-system + resource: vsphere.vm + filters: + - not: + - type: system + power_state: POWERED_ON + cpu_count: 1 + memory_size_mib: 1024 + """ + schema = type_schema( + 'system', + power_state={'type': 'string'}, + cpu_count={'type': 'number'}, + memory_size_mib={'type': 'number'}, + ) + + def process(self, resources, event=None): + results = [] + power_state = self.data.get('power_state', None) + cpu_count = self.data.get('cpu_count', None) + memory_size_mib = self.data.get('memory_size_mib', None) + for vm in resources: + matched = True + if power_state is not None and power_state != vm.get('power_state'): + matched = False + if cpu_count is not None and cpu_count > vm.get('cpu_count'): + matched = False + if memory_size_mib is not None and memory_size_mib > vm.get('memory_size_mib'): + matched = False + if matched: + results.append(vm) + return results \ No newline at end of file diff --git a/tools/c7n_vsphere/pyproject.toml b/tools/c7n_vsphere/pyproject.toml new file mode 100644 index 00000000000..7d86389a239 --- /dev/null +++ b/tools/c7n_vsphere/pyproject.toml @@ -0,0 +1,29 @@ +[tool.poetry] +name = "c7n_vsphere" +version = "1.0.0" +description = "Cloud Custodian - VMware vSphere Provider" +readme = "readme.md" +authors = ["Cloud Custodian Project"] +homepage = "https://cloudcustodian.io" +repository = "https://github.com/cloud-custodian/cloud-custodian" +documentation = "https://cloudcustodian.io/docs/" +license = "Apache-2.0" +classifiers = [ + "Topic :: System :: Systems Administration", + "Topic :: System :: Distributed Computing" +] + +[tool.poetry.dependencies] +python = "^3.7" +pyvmomi = "^7.0.2" +cryptography = "^2.9.2" + +[tool.poetry.dev-dependencies] +c7n = {path = "../..", develop = true} +pytest = "~6.0.0" +vcrpy = "^4.0.2" + + +[build-system] +requires = ["poetry>=0.12", "setuptools"] +build-backend = "poetry.masonry.api" \ No newline at end of file diff --git a/tools/c7n_vsphere/readme.md b/tools/c7n_vsphere/readme.md new file mode 100644 index 00000000000..deced11c37f --- /dev/null +++ b/tools/c7n_vsphere/readme.md @@ -0,0 +1,21 @@ +# Custodian VMware vSphere Support + +Work in Progress - Not Ready For Use. + +## Quick Start + +### Installation + +``` +pip install c7n_vsphere +``` + +## Examples + +filter examples: + +```yaml +policies: + - name: vsphere-vm + resource: vsphere.vm +``` diff --git a/tools/c7n_vsphere/requirements.txt b/tools/c7n_vsphere/requirements.txt new file mode 100644 index 00000000000..e751a20bdaa --- /dev/null +++ b/tools/c7n_vsphere/requirements.txt @@ -0,0 +1,6 @@ +git+git://github.com/vmware/vsphere-automation-sdk-python.git +pyvmomi==7.0.2 +lxml==4.6.3 +vmware-nsx==17.0.0 +suds-jurko==0.6 +oslo.vmware==3.9.0 \ No newline at end of file diff --git a/tools/c7n_vsphere/setup.py b/tools/c7n_vsphere/setup.py new file mode 100644 index 00000000000..58b2334bd71 --- /dev/null +++ b/tools/c7n_vsphere/setup.py @@ -0,0 +1,36 @@ +# Automatically generated from poetry/pyproject.toml +# flake8: noqa +# -*- coding: utf-8 -*- +from setuptools import setup + +packages = [ + 'c7n_vsphere', + 'c7n_vsphere.resources' +] + +package_data = {'': ['*']} + +install_requires = \ + ['pyvmomi (>=7.0.2)', + 'c7n (>=0.9.8,<0.10.0)'] + + +setup_kwargs = { + 'name': 'c7n-vsphere', + 'version': '1.0.0', + 'description': 'Cloud Custodian - VMware vSphere Provider', + 'long_description': '# Custodian VMware vSphere Support', + 'long_description_content_type': 'text/markdown', + 'author': 'Cloud Custodian Project', + 'author_email': None, + 'maintainer': None, + 'maintainer_email': None, + 'url': 'https://cloudcustodian.io', + 'packages': packages, + 'package_data': package_data, + 'install_requires': install_requires, + 'python_requires': '>=3.6,<4.0', +} + + +setup(**setup_kwargs) \ No newline at end of file diff --git a/tools/c7n_vsphere/test/vsphere.py b/tools/c7n_vsphere/test/vsphere.py new file mode 100644 index 00000000000..de43c360259 --- /dev/null +++ b/tools/c7n_vsphere/test/vsphere.py @@ -0,0 +1,225 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +import logging + +import requests +import urllib3 +from vmware.vapi.vsphere.client import create_vsphere_client + +from c7n.resources import load_resources + +session = requests.session() + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S') +# 本地测试用例 +def _loadFile_(): + json = dict() + f = open("/opt/fit2cloud/vsphere.txt") + lines = f.readlines() + for line in lines: + line = line.strip() + if "vsphere.VSPHERE_USERNAME" in line: + VSPHERE_USERNAME = line[line.rfind('=') + 1:] + json['VSPHERE_USERNAME'] = VSPHERE_USERNAME + if "vsphere.VSPHERE_PASSWORD" in line: + VSPHERE_PASSWORD = line[line.rfind('=') + 1:] + json['VSPHERE_PASSWORD'] = VSPHERE_PASSWORD + if "vsphere.VSPHERE_REGION_NAME" in line: + VSPHERE_REGION_NAME = line[line.rfind('=') + 1:] + json['VSPHERE_REGION_NAME'] = VSPHERE_REGION_NAME + if "vsphere.VSPHERE_SERVER" in line: + VSPHERE_SERVER = line[line.rfind('=') + 1:] + json['VSPHERE_SERVER'] = VSPHERE_SERVER + f.close() + print('认证信息: ' + str(json)) + return json + +params = _loadFile_() +load_resources() + +# Disable cert verification for demo purpose. +# This is not recommended in a production environment. +session.verify = False + +# Disable the secure connection warning for demo purpose. +# This is not recommended in a production environment. +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# Connect to a vCenter Server using username and password +vsphere_client = create_vsphere_client(server=params['VSPHERE_SERVER'], username=params['VSPHERE_USERNAME'], password=params['VSPHERE_PASSWORD'], session=session) + +def vm_list(): + vms = vsphere_client.vcenter.VM.list() + res = [] + for item in vms: + vm = vsphere_client.vcenter.VM.get(item.vm) + #{guest_os : CENTOS_7_64, name : FIT2CLOUD-2.0-TRY2, identity : None, power_state : POWERED_ON, instant_clone_frozen : None, + # hardware : {version : VMX_14, upgrade_policy : NEVER, upgrade_version : None, upgrade_status : NONE, upgrade_error : None}, + # boot : {type : BIOS, efi_legacy_boot : None, network_protocol : None, delay : 0, retry : False, retry_delay : 10, enter_setup_mode : False}, + # boot_devices : [], cpu : {count : 4, cores_per_socket : 1, hot_add_enabled : False, hot_remove_enabled : False}, + # memory : {size_mib : 16384, hot_add_enabled : False, hot_add_increment_size_mib : None, hot_add_limit_mib : None}, + # disks : {'2000': Info(label='Hard disk 1', type=HostBusAdapterType(string='SCSI'), ide=None, scsi=ScsiAddressInfo(bus=0, unit=0), sata=None, + # backing=BackingInfo(type=BackingType(string='VMDK_FILE'), vmdk_file='[Local] FIT2CLOUD-2.0-TRY2/FIT2CLOUD-2.0-TRY2-000001.vmdk'), capacity=107374182400)}, + # nics : {'4000': Info(label='Network adapter 1', type=EmulationType(string='VMXNET3'), upt_compatibility_enabled=True, + # mac_type=MacAddressType(string='GENERATED'), mac_address='00:0c:29:24:5d:a3', pci_slot_number=192, wake_on_lan_enabled=False, + # backing=BackingInfo(type=BackingType(string='STANDARD_PORTGROUP'), network='network-12', network_name='VM Network', host_device=None, + # distributed_switch_uuid=None, distributed_port=None, connection_cookie=None, opaque_network_type=None, opaque_network_id=None), + # state=ConnectionState(string='CONNECTED'), start_connected=True, allow_guest_control=True)}, cdroms : {'16000': Info(type=HostBusAdapterType(string='SATA'), + # label='CD/DVD drive 1', ide=None, sata=SataAddressInfo(bus=0, unit=0), backing=BackingInfo(type=BackingType(string='ISO_FILE'), + # iso_file='[Local] iso/CentOS-7-x86_64-Minimal-1804.iso', host_device=None, auto_detect=None, device_access_type=None), + # state=ConnectionState(string='NOT_CONNECTED'), start_connected=True, allow_guest_control=True)}, floppies : {}, parallel_ports : {}, serial_ports : {}, + # sata_adapters : {'15000': Info(label='SATA controller 0', type=Type(string='AHCI'), bus=0, pci_slot_number=35)}, + # scsi_adapters : {'1000': Info(label='SCSI controller 0', type=Type(string='PVSCSI'), scsi=ScsiAddressInfo(bus=0, unit=7), pci_slot_number=160, + # sharing=Sharing(string='NONE'))}} + data= { + "F2CId": item.vm, + "vm": item.vm, + "name": item.name, + "power_state": str(item.power_state), + "cpu_count": item.cpu_count, + "memory_size_mib": item.memory_size_mib, + "guest_os": str(vm.guest_os), + "identity": vm.identity, + "instant_clone_frozen": vm.instant_clone_frozen, + "hardware": str(vm.hardware), + "boot": str(vm.boot), + "boot_devices": vm.boot_devices, + "cpu": str(vm.cpu), + "memory": str(vm.memory), + "disks": str(vm.disks), + } + res.append(data) + print(res) + return res + +def cluster_list(): + clusters = vsphere_client.vcenter.Cluster.list() + res = [] + for item in clusters: + cluster = vsphere_client.vcenter.Cluster.get(item.cluster) + data= { + "F2CId": item.cluster, + "cluster": item.cluster, + "name": item.name, + "ha_enabled": item.ha_enabled, + "drs_enabled": item.drs_enabled, + "resource_pool": cluster.resource_pool + } + res.append(data) + print(res) + return res + +def datacenter_list(): + datacenters = vsphere_client.vcenter.Datacenter.list() + res = [] + for item in datacenters: + #{name : Datacenter, datastore_folder : group-s5, host_folder : group-h4, network_folder : group-n6, vm_folder : group-v3} + datacenter = vsphere_client.vcenter.Datacenter.get(item.datacenter) + print(datacenter) + data= { + "F2CId": item.datacenter, + "datacenter": item.datacenter, + "name": item.name, + "datastore_folder": datacenter.datastore_folder, + "host_folder": datacenter.host_folder, + "network_folder": datacenter.network_folder, + "vm_folder": datacenter.vm_folder, + } + res.append(data) + return res + +def datastore_list(): + datastores = vsphere_client.vcenter.Datastore.list() + res = [] + for item in datastores: + #{name : datastore1, type : VMFS, accessible : True, free_space : 610536521728, multiple_host_access : False, thin_provisioning_supported : True} + datastore = vsphere_client.vcenter.Datastore.get(item.datastore) + print(datastore) + data= { + "F2CId": item.datastore, + "datastore": item.datastore, + "name": item.name, + "type": str(item.type), + "free_space": item.free_space, + "capacity": item.capacity, + "accessible": datastore.accessible, + "multiple_host_access": datastore.multiple_host_access, + "thin_provisioning_supported": datastore.thin_provisioning_supported, + } + res.append(data) + return res + +def folder_list(): + folders = vsphere_client.vcenter.Folder.list() + #Summary(folder='group-d1', name='Datacenters', type=Type(string='DATACENTER')) + res = [] + for item in folders: + data= { + "F2CId": item.folder, + "folder": item.folder, + "name": item.name, + "type": str(item.type), + } + res.append(data) + print(res) + return folders + +def host_list(): + hosts = vsphere_client.vcenter.Host.list() + #Summary(host='host-10', name='10.1.240.15', connection_state=ConnectionState(string='CONNECTED'), power_state=PowerState(string='POWERED_ON')) + res = [] + for item in hosts: + data= { + "F2CId": item.host, + "host": item.host, + "name": item.name, + "connection_state": str(item.connection_state), + "power_state": str(item.power_state), + } + res.append(data) + print(res) + return res + +def network_list(): + networks = vsphere_client.vcenter.Network.list() + #Summary(network='dvportgroup-1312', name='cluster-try-VSAN-DPortGroup', type=Type(string='DISTRIBUTED_PORTGROUP')) + res = [] + for item in networks: + data= { + "F2CId": item.network, + "network": item.network, + "name": item.name, + "type": str(item.type), + } + res.append(data) + print(res) + return res + +def resource_pool_list(): + resource_pools = vsphere_client.vcenter.ResourcePool.list() + #Summary(resource_pool='resgroup-34', name='Resources') + res = [] + for item in resource_pools: + #{name : Resources, resource_pools : set(), cpu_allocation : None, memory_allocation : None} + resource_pool = vsphere_client.vcenter.ResourcePool.get(item.resource_pool) + data= { + "F2CId": item.resource_pool, + "resource_pool": item.resource_pool, + "name": item.name, + "cpu_allocation": resource_pool.cpu_allocation, + "memory_allocation": resource_pool.memory_allocation + } + res.append(data) + print(res) + return res + +if __name__ == '__main__': + logging.info("Hello vSphere OpenApi!") + vm_list() + # cluster_list() + # datacenter_list() + # datastore_list() + # folder_list() + # host_list() + # network_list() + # resource_pool_list() \ No newline at end of file diff --git a/tools/dev/dockerpkg.py b/tools/dev/dockerpkg.py index 33d73e9eaee..c1f94714957 100644 --- a/tools/dev/dockerpkg.py +++ b/tools/dev/dockerpkg.py @@ -63,15 +63,19 @@ RUN rm -R tools/c7n_azure/tests_azure ADD tools/c7n_kube /src/tools/c7n_kube RUN rm -R tools/c7n_kube/tests -ADD tools/c7n_aliyun /src/tools/c7n_aliyun +ADD tools/c7n_vsphere /src/tools/c7n_aliyun RUN rm -R tools/c7n_aliyun/test ADD tools/c7n_huawei /src/tools/c7n_huawei RUN rm -R tools/c7n_huawei/test ADD tools/c7n_tencent /src/tools/c7n_tencent RUN rm -R tools/c7n_tencent/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test +ADD tools/c7n_vsphere /src/tools/c7n_vsphere +RUN rm -R tools/c7n_vsphere/test # Install requested providers -ARG providers="azure gcp kube aliyun huawei tencent" +ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere" RUN . /usr/local/bin/activate && \\ for pkg in $providers; do cd tools/c7n_$pkg && \\ $HOME/.poetry/bin/poetry install && cd ../../; done