feat(build): 修改阿里云镜像仓库地址。

Makefile

@@ -1,6 +1,6 @@
 SELF_MAKE := $(lastword $(MAKEFILE_LIST))
 
-PKG_SET = tools/c7n_tencent tools/c7n_huawei tools/c7n_aliyun tools/c7n_gcp tools/c7n_azure tools/c7n_kube tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext
+PKG_SET = tools/c7n_tencent tools/c7n_huawei tools/c7n_aliyun tools/c7n_gcp tools/c7n_azure tools/c7n_kube tools/c7n_openstack tools/c7n_vsphere tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext
 
 install:
 	python3 -m venv .

build.sh

@@ -1,4 +1,4 @@
 echo "构建custodian镜像 ..."
 
-docker build -t registry.cn-qingdao.aliyuncs.com/x-lab/riskscanner/custodian:master .
-docker push registry.cn-qingdao.aliyuncs.com/x-lab/riskscanner/custodian:master
+docker build -t registry.cn-qingdao.aliyuncs.com/x-lab/custodian:1.4 .
+docker push registry.cn-qingdao.aliyuncs.com/x-lab/custodian:1.4

c7n/commands.py

@@ -553,4 +553,14 @@ def version_cmd(options):
         packages.append('c7n_azure')
     if 'k8s' in found:
         packages.append('c7n_kube')
+    if 'aliyun' in found:
+        packages.append('c7n_aliyun')
+    if 'tencent' in found:
+        packages.append('c7n_tencent')
+    if 'huawei' in found:
+        packages.append('c7n_huawei')
+    if 'openstack' in found:
+        packages.append('c7n_openstack')
+    if 'vsphere' in found:
+        packages.append('c7n_vsphere')
     print(generate_requirements(packages))

c7n/resources/__init__.py

@@ -49,7 +49,7 @@ def should_load_provider(name, provider_types):
     return False
 
 
-PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'aliyun', 'huawei', 'tencent')
+PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'aliyun', 'huawei', 'tencent', 'openstack', 'vsphere')
 
 
 def load_available(resources=True):
@@ -73,7 +73,6 @@ def load_available(resources=True):
 
 def load_providers(provider_types):
     global LOADED
-
     # Even though we're lazy loading resources we still need to import
     # those that are making available generic filters/actions
     if should_load_provider('aws', provider_types):
@@ -108,4 +107,11 @@ def load_providers(provider_types):
         from c7n_tencent.entry import initialize_tencent
         initialize_tencent()
 
+    if should_load_provider('openstack', provider_types):
+        from c7n_openstack.entry import initialize_openstack
+        initialize_openstack()
+
+    if should_load_provider('vsphere', provider_types):
+        from c7n_vsphere.entry import initialize_vsphere
+        initialize_vsphere()
     LOADED.update(provider_types)

docker/cli

@@ -1,4 +1,4 @@
-FROM registry.fit2cloud.com/fit2cloud2/fabric8-java-alpine-openjdk8-jre:py37 as build-env
+FROM registry.cn-qingdao.aliyuncs.com/x-lab/fabric8-java-alpine-openjdk8-jre:py37 as build-env
 
 USER root
 
@@ -8,7 +8,7 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 # pre-requisite distro deps, and build env setup
 RUN apk add --update \
  curl \
- && apk add gcc g++ make libffi-dev openssl-dev libtool --no-cache \
+ && apk add gcc g++ make git libffi-dev openssl-dev libxml2-dev libxslt-dev musl-dev cargo libtool --no-cache \
  && python3 -m venv /usr/local \
  && curl -sSL https://nginx-qa.fit2cloud.com/tools/get-poetry.py | python3
 
@@ -26,31 +26,37 @@ ADD tools/c7n_gcp /src/tools/c7n_gcp
 RUN rm -R tools/c7n_gcp/tests
 ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
-ADD tools/c7n_kube /src/tools/c7n_kube
-RUN rm -R tools/c7n_kube/tests
+#ADD tools/c7n_kube /src/tools/c7n_kube
+#RUN rm -R tools/c7n_kube/tests
 ADD tools/c7n_aliyun /src/tools/c7n_aliyun
 RUN rm -R tools/c7n_aliyun/test
 ADD tools/c7n_huawei /src/tools/c7n_huawei
 RUN rm -R tools/c7n_huawei/test
 ADD tools/c7n_tencent /src/tools/c7n_tencent
 RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube aliyun huawei tencent"
+ARG providers="azure gcp aliyun huawei tencent openstack vsphere"
+RUN . /usr/local/bin/activate && \
+    pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry lock && \
     $HOME/.poetry/bin/poetry install && cd ../../; done
 
 RUN mkdir /output
 
-FROM registry.fit2cloud.com/fit2cloud2/fabric8-java-alpine-openjdk8-jre:py37
+FROM registry.cn-qingdao.aliyuncs.com/x-lab/fabric8-java-alpine-openjdk8-jre:py37
 
 COPY --from=build-env /src /src
 COPY --from=build-env /usr/local /usr/local
 COPY --from=build-env /output /output
 
-RUN rm -Rf /var/cache/apk
+RUN rm -Rf /var/cache/apk/*
 
 USER root
 WORKDIR /home/custodian

docker/cli-distroless

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/mailer

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/mailer-distroless

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/org

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/org-distroless

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/policystream

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

docker/policystream-distroless

@@ -25,9 +25,19 @@ ADD tools/c7n_azure /src/tools/c7n_azure
 RUN rm -R tools/c7n_azure/tests_azure
 ADD tools/c7n_kube /src/tools/c7n_kube
 RUN rm -R tools/c7n_kube/tests
+ADD tools/c7n_aliyun /src/tools/c7n_aliyun
+RUN rm -R tools/c7n_aliyun/test
+ADD tools/c7n_huawei /src/tools/c7n_huawei
+RUN rm -R tools/c7n_huawei/test
+ADD tools/c7n_tencent /src/tools/c7n_tencent
+RUN rm -R tools/c7n_tencent/test
+ADD tools/c7n_openstack /src/tools/c7n_openstack
+RUN rm -R tools/c7n_openstack/test
+ADD tools/c7n_vsphere /src/tools/c7n_vsphere
+RUN rm -R tools/c7n_vsphere/test
 
 # Install requested providers
-ARG providers="azure gcp kube"
+ARG providers="azure gcp kube aliyun huawei tencent openstack vsphere"
 RUN . /usr/local/bin/activate && \
     for pkg in $providers; do cd tools/c7n_$pkg && \
     $HOME/.poetry/bin/poetry install && cd ../../; done

requirements-dev.txt

@@ -8,6 +8,10 @@
 -r tools/c7n_mailer/requirements.txt
 -r tools/c7n_org/requirements.txt
 -r tools/c7n_aliyun/requirements.txt
+-r tools/c7n_huawei/requirements.txt
+-r tools/c7n_tencent/requirements.txt
+-r tools/c7n_openstack/requirements.txt
+-r tools/c7n_vsphere/requirements.txt
 
 # Setup source directories as editable/development distributions
 -e .
@@ -23,6 +27,14 @@
 -e tools/c7n_org
 # Local package required for c7n_aliyun tests
 -e tools/c7n_aliyun
+# Local package required for c7n_huawei tests
+-e tools/c7n_huawei
+# Local package required for c7n_tencent tests
+-e tools/c7n_tencent
+# Local package required for c7n_vsphere tests
+-e tools/c7n_openstack
+# Local package required for c7n_vsphere tests
+-e tools/c7n_vsphere
 
 # we don't export dev requirements of subpackages (due to editable/distribution above)
 # so explicitly list them here..

tests/test_docker.py

@@ -162,7 +162,7 @@ def test_image_metadata(image_name):
 def test_cli_providers_available():
     providers = os.environ.get("CUSTODIAN_PROVIDERS", None)
     if providers is None:
-        providers = {"aws", "azure", "gcp", "k8s"}
+        providers = {"aws", "azure", "gcp", "k8s", "aliyun", "huawei", "tencent", "openstack", "vsphere"}
     elif providers == "":
         providers = {"aws"}
     else:

tests/test_packaging.py

@@ -19,7 +19,8 @@
 @pytest.mark.parametrize("package", [
     "c7n", "c7n_azure", "c7n_gcp", "c7n_kube", "c7n_org",
     "c7n_mailer", "policystream", "c7n_trailcreator",
-    "c7n_logexporter", "c7n_sphinxext"])
+    "c7n_logexporter", "c7n_sphinxext", "c7n_aliyun",
+    "c7n_huawei", "c7n_tencent", "c7n_vsphere", "c7n_vsphere"])
 def test_package_metadata(package):
     try:
         m = __import__(package)

tools/c7n_aliyun/.gitignore

@@ -2,3 +2,4 @@
 *py~
 __pycache__
 *.egg-info
+aliyun.py

tools/c7n_aliyun/c7n_aliyun/client.py

@@ -61,26 +61,26 @@ def get_oss_region(self, regionId):
         return region
 
 REGION_ENDPOINT = {
-        'cn-hangzhou': 'oss-cn-hangzhou.aliyuncs.com',
-        'cn-shanghai': 'oss-cn-shanghai.aliyuncs.com',
-        'cn-qingdao': 'oss-cn-qingdao.aliyuncs.com',
-        'cn-beijing': 'oss-cn-beijing.aliyuncs.com',
-        'cn-zhangjiakou': 'oss-cn-zhangjiakou.aliyuncs.com',
-        'cn-huhehaote': 'oss-cn-huhehaote.aliyuncs.com',
-        'cn-shenzhen': 'oss-cn-shenzhen.aliyuncs.com',
-        'cn-hongkong': 'oss-cn-hongkong.aliyuncs.com',
-        'us-west-1': 'oss-us-west-1.aliyuncs.com',
-        'us-east-1': 'oss-us-east-1.aliyuncs.com',
-        'ap-southeast-1': 'oss-ap-southeast-1.aliyuncs.com',
-        'ap-southeast-2': 'oss-ap-southeast-2.aliyuncs.com',
-        'ap-southeast-3': 'oss-ap-southeast-3.aliyuncs.com',
-        'ap-southeast-5': 'oss-ap-southeast-5.aliyuncs.com',
-        'ap-northeast-1': 'oss-ap-northeast-1.aliyuncs.com',
-        'ap-south-1': 'oss-ap-south-1.aliyuncs.com',
-        'eu-central-1': 'oss-eu-central-1.aliyuncs.com',
-        'eu-west-1': 'oss-eu-west-1.aliyuncs.com',
-        'me-east-1': 'oss-me-east-1.aliyuncs.com',
-        'cn-wulanchabu': 'oss-cn-wulanchabu.aliyuncs.com',
-        'cn-heyuan': 'oss-cn-heyuan.aliyuncs.com',
-        'cn-chengdu': 'oss-cn-chengdu.aliyuncs.com'
-    }
+    'cn-hangzhou': 'oss-cn-hangzhou.aliyuncs.com',
+    'cn-shanghai': 'oss-cn-shanghai.aliyuncs.com',
+    'cn-qingdao': 'oss-cn-qingdao.aliyuncs.com',
+    'cn-beijing': 'oss-cn-beijing.aliyuncs.com',
+    'cn-zhangjiakou': 'oss-cn-zhangjiakou.aliyuncs.com',
+    'cn-huhehaote': 'oss-cn-huhehaote.aliyuncs.com',
+    'cn-shenzhen': 'oss-cn-shenzhen.aliyuncs.com',
+    'cn-hongkong': 'oss-cn-hongkong.aliyuncs.com',
+    'us-west-1': 'oss-us-west-1.aliyuncs.com',
+    'us-east-1': 'oss-us-east-1.aliyuncs.com',
+    'ap-southeast-1': 'oss-ap-southeast-1.aliyuncs.com',
+    'ap-southeast-2': 'oss-ap-southeast-2.aliyuncs.com',
+    'ap-southeast-3': 'oss-ap-southeast-3.aliyuncs.com',
+    'ap-southeast-5': 'oss-ap-southeast-5.aliyuncs.com',
+    'ap-northeast-1': 'oss-ap-northeast-1.aliyuncs.com',
+    'ap-south-1': 'oss-ap-south-1.aliyuncs.com',
+    'eu-central-1': 'oss-eu-central-1.aliyuncs.com',
+    'eu-west-1': 'oss-eu-west-1.aliyuncs.com',
+    'me-east-1': 'oss-me-east-1.aliyuncs.com',
+    'cn-wulanchabu': 'oss-cn-wulanchabu.aliyuncs.com',
+    'cn-heyuan': 'oss-cn-heyuan.aliyuncs.com',
+    'cn-chengdu': 'oss-cn-chengdu.aliyuncs.com'
+}