Merge pull request #56 from cxl123156/br_release_sdk_v3

add rds cpu&mem field, support input ak&sk from command line

README.md

@@ -33,6 +33,22 @@ The default port is 8087, default config file location is ./clouds.yml.
 Visit metrics in http://localhost:8087/metrics?services=SYS.VPC,SYS.ELB
 
 
+To avoid ak&sk leaks, you can input your ak&sk through the command line with '-s' param like this
+```shell
+./cloudeye-exporter -s true
+```
+To startup the cloudeye-exporter using script, here is an example
+```shell
+#!/bin/bash
+# Don't write plain text ak&sk in script.
+# Encrypt the ak&sk instead and use your own decrypt function to assign the return value.
+huaweiCloud_AK=your_decrypt_function("your encrypted ak")
+huaweiCloud_SK=your_decrypt_function("your encrypted sk")
+$(./cloudeye-exporter -s true<<EOF
+$huaweiCloud_AK $huaweiCloud_SK
+EOF)
+```
+
 ## Help
 ```
 Usage of ./cloudeye-exporter:
@@ -41,7 +57,7 @@ Usage of ./cloudeye-exporter:
 ```
 
 ## Example of config file(clouds.yml)
-The "URL" value can be get from [Identity and Access Management (IAM) endpoint list](https://developer.huaweicloud.com/endpoint?IAM).
+The "URL" value can be get from [Identity and Access Management (IAM) endpoint list (Internal)](https://developer.huaweicloud.com/intl/en-us/endpoint?IAM) and [Identity and Access Management (IAM) endpoint list (China)](https://developer.huaweicloud.com/endpoint?IAM).
 ```
 global:
   prefix: "huaweicloud"

README_cn.md

@@ -60,9 +60,11 @@ Prometheus是用于展示大型测量数据的开源可视化工具，在工业
 | ECS | Ubuntu 18.04 |
 | Ubuntu private ip | 192.168.0.xx |
 
+账号要求具有IAM，CES，RMS，EPS服务的可读权限,另外获取哪些服务的监控数据就需要有哪些服务的只读权限
+
 ## 安装配置cloudeye-exporter
 1. 在ubuntu vm上安装cloudeye-exporter
-   
+
    登录vm机器，查看插件Releases版本 (https://github.com/huaweicloud/cloudeye-exporter/releases) ，获取插件下载地址，下载解压安装。
 ```
 # 参考命令：
@@ -72,22 +74,41 @@ wget https://github.com/huaweicloud/cloudeye-exporter/releases/download/v2.0.5/c
 tar -xzvf cloudeye-exporter.v2.0.5.tar.gz
 ```
 2. 编辑clouds.yml文件配置公有云信息
+
+   区域ID以及auth_url可点击下面链接查看
+ *  [地区和终端节点（中国站）](https://developer.huaweicloud.com/endpoint?IAM)
+ *  [地区和终端节点（国际站）](https://developer.huaweicloud.com/intl/en-us/endpoint?IAM)
 ```
 global:
-  port: ":8087"
+  port: ":8087" # 监听端口 :8087代表在全部网络接口上开启监听8087端口，限定内部访问也可以指定IP例如：192.168.1.100:8087
   scrape_batch_size: 300
 auth:
   auth_url: "https://iam.{region_id}.myhuaweicloud.com/v3"
-  project_name: "cn-north-1"
-  access_key: ""
+  project_name: "cn-north-1" # 华为云项目名称，可以在“华为云->统一身份认证服务->项目”中查看
+  access_key: "" # IAM用户访问密钥 您可参考3.1章节使用命令行输入加密后的ak sk，避免在配置文件中明文配置AK SK
   secret_key: ""
-  region: "cn-north-1"
+  region: "cn-north-1" # 区域ID
 ```
 注：默认的监控端口为8087.
 
-3. 启动cloudeye-exporter
+3. 启动cloudeye-exporter，默认读取当前目录下的clouds.yml文件，也可使用-config参数指定clouds.yml文件路径
+```
+./cloudeye-exporter -config=clouds.yml
+```
+
+3.1 出于安全考虑cloudeye-exporter提供了 -s参数, 可以通过命令行交互的方式输入ak sk避免明文配置在clouds.yml文件中引起泄露
+```shell
+./cloudeye-exporter -s true
 ```
-./cloudeye-exporter
+下面是shell脚本启动的样例，建议在脚本中配置加密后的ak&sk，并通过您自己的解密方法对ak sk进行解密后通过huaweiCloud_AK和huaweiCloud_SK参数传入cloudeye-exporter
+```shell
+#!/bin/bash
+## 为了防止您的ak&sk泄露，不建议在脚本中配置明文的ak sk
+huaweiCloud_AK=your_decrypt_function("加密的AK")
+huaweiCloud_SK=your_decrypt_function("加密的SK")
+$(./cloudeye-exporter -s true<<EOF
+$huaweiCloud_AK $huaweiCloud_SK
+EOF)
 ```
 
 ## 安装配置prometheus接入cloudeye
@@ -103,14 +124,14 @@ $ cd prometheus-2.14.0.linux-amd64
    ```
    $ vi prometheus.yml
    global:
-     scrape_interval: 1m # Set the scrape interval to every 1 minute seconds. Default is every 1 minute.
-     scrape_timeout: 1m
+     scrape_interval: 1m # 设置prometheus从exporter查询数据的间隔时间，prometheus配置文件中默认为15s，建议设置为1m
+     scrape_timeout: 1m # 设置从exporter查询数据的超时时间，prometheus配置文件中默认为15s，建议设置为1m
    scrape_configs:
      - job_name: 'huaweicloud'
        static_configs:
-       - targets: ['192.168.0.xx:8087']
+       - targets: ['192.168.0.xx:8087'] # exporter节点地址:监听端口
        params:
-         services: ['SYS.VPC,SYS.RDS']
+         services: ['SYS.VPC,SYS.RDS'] # 当前任务需要查询的服务命名空间，建议每个服务配置单独job
    ```
 3. 启动prometheus监控华为云服务
 ```

collector/apic.go

@@ -69,7 +69,7 @@ func buildApisInfo(instanceId string, resourceInfos map[string]labelInfo, filter
 		*filterMetrics = append(*filterMetrics, metrics...)
 		appInfo := labelInfo{
 			Name:  []string{"appName", "groupName", "groupId", "reqMethod", "reqUri"},
-			Value: []string{api.Name, getDefaultString(api.GroupName), getDefaultString(api.GroupName), api.ReqMethod.Value(), api.ReqUri},
+			Value: []string{api.Name, getDefaultString(api.GroupName), api.GroupId, api.ReqMethod.Value(), api.ReqUri},
 		}
 		appInfo.Name = append(appInfo.Name, instanceInfo.Name...)
 		appInfo.Value = append(appInfo.Value, instanceInfo.Value...)

collector/apic_test.go

@@ -0,0 +1,116 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/apig/v2/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShowDetailsOfInstanceV2(t *testing.T) {
+	apigClient := getAPICSClient()
+	id := "0001-0001-0000001"
+	name := "instance01"
+	instanceInfo := model.ShowDetailsOfInstanceV2Response{
+		Id:           &id,
+		InstanceName: &name,
+	}
+	patches := gomonkey.ApplyMethodReturn(apigClient, "ShowDetailsOfInstanceV2", &instanceInfo, nil)
+	defer patches.Reset()
+	resp, err := showDetailsOfInstanceV2(id)
+	assert.Equal(t, true, err == nil)
+	assert.Equal(t, name, *resp.InstanceName)
+}
+
+func TestGetResourceInfo(t *testing.T) {
+	var (
+		id       = "0001-0001-000001"
+		name     = "instance01"
+		eip      = "*.*.*.*"
+		epId     = "0"
+		apiId    = "api0000001"
+		emptyStr = ""
+	)
+
+	sysConfig := map[string][]string{
+		"instance_id":         {"requests"},
+		"instance_id,api_id":  {"req_count"},
+		"instance_id,node_ip": {"node_qps"},
+	}
+
+	instances := []model.RespInstanceBase{{Id: &id, InstanceName: &name, EipAddress: &eip, EnterpriseProjectId: &epId}}
+	apis := []model.ApiInfoPerPage{{Id: &apiId, Name: "api1", GroupName: &emptyStr, GroupId: "group1"}}
+	instance := model.ShowDetailsOfInstanceV2Response{
+		NodeIps: &model.NodeIps{
+			Livedata: &[]string{"*.*.*.2"},
+			Shubao:   &[]string{"*.*.*.3"},
+		},
+	}
+
+	patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyFuncReturn(getAllAPICInstances, instances, nil)
+	patches.ApplyFuncReturn(getApisOfInstances, apis, nil)
+	patches.ApplyFuncReturn(showDetailsOfInstanceV2, &instance, nil)
+	defer patches.Reset()
+	var getter = APICInfo{}
+	label, metrics := getter.GetResourceInfo()
+	assert.Equal(t, 4, len(label))
+	assert.Equal(t, 4, len(metrics))
+}
+
+func TestGetAllAPICInstances(t *testing.T) {
+	var (
+		id   = "0001-0001-000001"
+		name = "instance01"
+	)
+	respPage1 := model.ListInstancesV2Response{
+		HttpStatusCode: 200,
+		Instances: &[]model.RespInstanceBase{
+			{Id: &id, InstanceName: &name},
+		},
+	}
+	respPage2 := model.ListInstancesV2Response{
+		HttpStatusCode: 200,
+		Instances:      &[]model.RespInstanceBase{},
+	}
+	apicClient := getAPICSClient()
+	patches := gomonkey.ApplyMethodFunc(apicClient, "ListInstancesV2", func(req *model.ListInstancesV2Request) (*model.ListInstancesV2Response, error) {
+		if *req.Offset == 0 {
+			return &respPage1, nil
+		}
+		return &respPage2, nil
+	})
+	defer patches.Reset()
+	instances, err := getAllAPICInstances()
+	assert.Equal(t, true, err == nil)
+	assert.Equal(t, 1, len(instances))
+}
+
+func TestGetApisOfInstances(t *testing.T) {
+	var (
+		id   = "0001-0001-000001"
+		name = "api01"
+	)
+	respPage1 := model.ListApisV2Response{
+		HttpStatusCode: 200,
+		Apis: &[]model.ApiInfoPerPage{
+			{Id: &id, Name: name},
+		},
+	}
+	respPage2 := model.ListApisV2Response{
+		HttpStatusCode: 200,
+		Apis:           &[]model.ApiInfoPerPage{},
+	}
+	apicClient := getAPICSClient()
+	patches := gomonkey.ApplyMethodFunc(apicClient, "ListApisV2", func(req *model.ListApisV2Request) (*model.ListApisV2Response, error) {
+		if *req.Offset == 0 {
+			return &respPage1, nil
+		}
+		return &respPage2, nil
+	})
+	defer patches.Reset()
+	apis, err := getApisOfInstances("00001")
+	assert.Equal(t, true, err == nil)
+	assert.Equal(t, 1, len(apis))
+}

collector/apig_test.go

@@ -0,0 +1,43 @@
+package collector
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/def"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestApiGGetResourceInfo(t *testing.T) {
+	respPage1 := ListAppsResponse{
+		HttpStatusCode: 200,
+		Apps: &[]Apps{
+			{ID: "app-0001", Name: "app1"},
+		},
+	}
+	respPage2 := ListAppsResponse{
+		HttpStatusCode: 200,
+		Apps:           &[]Apps{},
+	}
+	sysConfig := map[string][]string{"api_id": {"req_count"}}
+
+	apigClient := getAPIGSClient()
+	patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyMethodFunc(apigClient.HcClient, "Sync", func(req interface{}, reqDef *def.HttpRequestDef) (interface{}, error) {
+		request, ok := req.(*ListAppsRequest)
+		if !ok {
+			return nil, errors.New("test error")
+		}
+		if *request.Offset == 0 {
+			return &respPage1, nil
+		}
+		return &respPage2, nil
+	})
+	defer patches.Reset()
+
+	var getter APIGInfo
+	labels, metrics := getter.GetResourceInfo()
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, 1, len(metrics))
+}

collector/as_test.go

@@ -0,0 +1,23 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAsGetResourceInfo(t *testing.T) {
+	sysConfig := map[string][]string{"AutoScalingGroup": {"req_count"}}
+	groups := []ResourceBaseInfo{
+		{ID: "0001-0001-000000001", Name: "group01", EpId: "0"},
+	}
+	patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyFuncReturn(getResourcesBaseInfoFromRMS, groups, nil)
+	defer patches.Reset()
+
+	var asGetter ASInfo
+	labels, metrics := asGetter.GetResourceInfo()
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, 1, len(metrics))
+}

collector/bms_test.go

@@ -0,0 +1,29 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBmsGetResourceInfo(t *testing.T) {
+	sysConfig := map[string][]string{"instance_id": {"cpu_utils"}}
+	instances := []EcsInstancesInfo{
+		{ResourceBaseInfo: ResourceBaseInfo{ID: "0001-0001-000000001", Name: "host01", EpId: "0"}},
+	}
+	patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyFuncReturn(getAllServerFromRMS, instances, nil)
+	patches.ApplyFunc(loadAgentDimensions, func(_ string) { return })
+	patches.ApplyFuncReturn(getIPFromEcsInfo, "")
+	defer patches.Reset()
+
+	var bmsGetter BMSInfo
+	labels, metrics := bmsGetter.GetResourceInfo()
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, 1, len(metrics))
+
+	var servicesGetter SERVICEBMSInfo
+	serviceLabel, _ := servicesGetter.GetResourceInfo()
+	assert.Equal(t, 1, len(serviceLabel))
+}

collector/cbr_test.go

@@ -0,0 +1,40 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cbr/v1/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCbrGetResourceInfo(t *testing.T) {
+	defaultEpId := "0"
+	respPage1 := model.ListVaultResponse{
+		HttpStatusCode: 200,
+		Vaults: &[]model.Vault{
+			{Id: "app-0001", Name: "app1", EnterpriseProjectId: &defaultEpId},
+		},
+	}
+	respPage2 := model.ListVaultResponse{
+		HttpStatusCode: 200,
+		Vaults:         &[]model.Vault{},
+	}
+	sysConfig := map[string][]string{"instance_id": {"vault_util"}}
+
+	cbrClient := getCBRClient()
+	patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyFuncReturn(getResourceFromRMS, false)
+	patches.ApplyMethodFunc(cbrClient, "ListVault", func(req *model.ListVaultRequest) (*model.ListVaultResponse, error) {
+		if *req.Offset == 0 {
+			return &respPage1, nil
+		}
+		return &respPage2, nil
+	})
+	defer patches.Reset()
+
+	var cbrgetter CBRInfo
+	labels, metrics := cbrgetter.GetResourceInfo()
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, 1, len(metrics))
+}