发布2.0.10版本 (#66)

1.清理已有模板中的copyright链接
2.新增gaussdb for mysql服务grafana模
3.适配geminidb redis单维度指标
4.logs.yml文件路径支持可配置
5.补充NAT服务下的指标

README.md

@@ -61,15 +61,19 @@ The "URL" value can be get from [Identity and Access Management (IAM) endpoint l
 ```
 global:
   prefix: "huaweicloud"
-  port: ":8087"
+  port: "{private IP}:8087" # For security purposes, you are advised not to expose the Expoter service port to the public network. You are advised to set this parameter to 127.0.0.1:{port} or {private IP address}:{port}, for example, 192.168.1.100:8087. If the port needs to be exposed to the public network, ensure that the security group, firewall, and iptables access control policies are properly configured to meet the minimum access permission principle.
   metric_path: "/metrics"
   scrape_batch_size: 300
   resource_sync_interval_minutes: 20 # Update frequency of resource information: resource information is updated every 180 minutes by default; If this parameter is set to a value less than 10 minutes, the information is updated every 10 minutes.
   ep_ids: "xxx1,xxx2" # This is optional. Filter resources by enterpries project, cloudeye-exporter will get all resources when this is empty, if you need multiple enterprise project, use comma split them.
+  logs_conf_path: "/root/logs.yml" # This is optional. We recommend that you use an absolute path for the log configuration file path. If this line is absent, the program will use configuration file in the directory where the startup command is executed by default.
+  metrics_conf_path: "/root/metrics.yml" # This is optional. We recommend that you use an absolute path for the metrics configuration file path. If this line is absent, the program will use configuration file in the directory where the startup command is executed by default.
+  endpoints_conf_path: "/root/endpoints.yml" # This is optional. We recommend that you use an absolute path for the service endpoints configuration file path. If this line is absent, the program will use configuration file in the directory where the startup command is executed by default.
+  ignore_ssl_verify: false # This is optional. The SSL certificate is verified by default when the exporter queries resources or indicators. If the exporter is abnormal due to SSL certificate verification, you can set this configuration to true to skip SSL certificate verification.
 auth:
   auth_url: "https://iam.{region_id}.myhuaweicloud.com/v3"
   project_name: "{project_name}"
-  access_key: "{access_key}"
+  access_key: "{access_key}" # It is strongly remommended that you use a script to decrypt the AK/SK by following the instructions provided in section 4.1 to prevent information leakage caused by plaintext AK/SK configuration in the configuration file.
   secret_key: "{secret_key}"
   region: "{region}"
 ```

README_cn.md

@@ -81,14 +81,18 @@ tar -xzvf cloudeye-exporter.v2.0.5.tar.gz
  *  [地区和终端节点（国际站）](https://developer.huaweicloud.com/intl/en-us/endpoint?IAM)
 ```
 global:
-  port: ":8087" # 监听端口 :8087代表在全部网络接口上开启监听8087端口，限定内部访问也可以指定IP例如：192.168.1.100:8087
+  port: "{private IP}:8087" # 监听端口 :出于安全考虑，建议不将expoter服务端口暴露到公网，建议配置为127.0.0.1:{port}，或{内网ip}:{port}，例如：192.168.1.100:8087；如业务需要将该端口暴露到公网，请确保合理配置安全组，防火墙，iptables等访问控制策略，确保最小访问权限
   scrape_batch_size: 300
   resource_sync_interval_minutes: 20 # 资源信息更新频率：默认180分钟更新一次；该配置值小于10分钟，将以10分钟1次为资源信息更新频率
   ep_ids: "xxx1,xxx2" # 可选配置，根据企业项目ID过滤资源，不配置默认查询所有资源的指标，多个ID使用英文逗号进行分割。
+  logs_conf_path: "/root/logs.yml" # 可选配置，指定日志打印配置文件路径，建议使用绝对路径。若未指定，程序将默认使用执行启动命令所在目录下的日志配置文件。
+  metrics_conf_path: "/root/metric.yml" # 可选配置，指定指标配置文件路径，建议使用绝对路径。若未指定，程序将默认使用执行启动命令所在目录下的指标配置文件。
+  endpoints_conf_path: "/root/endpoints.yml" # 可选配置，指定服务域名配置文件路径，建议使用绝对路径。若未指定，程序将默认使用执行启动命令所在目录下的服务域名配置文件。
+  ignore_ssl_verify: false # 可选配置，exporter查询资源/指标时默认校验ssl证书；若用户因ssl证书校验导致功能异常，可将该配置项配置为true跳过ssl证书校验
 auth:
   auth_url: "https://iam.{region_id}.myhuaweicloud.com/v3"
   project_name: "cn-north-1" # 华为云项目名称，可以在“华为云->统一身份认证服务->项目”中查看
-  access_key: "" # IAM用户访问密钥 您可参考3.1章节使用命令行输入加密后的ak sk，避免在配置文件中明文配置AK SK
+  access_key: "" # IAM用户访问密钥 您可参考4.1章节，使用脚本将ak sk解密后传入，避免因在配置文件中明文配置AK SK而引发信息泄露
   secret_key: ""
   region: "cn-north-1" # 区域ID
 ```
@@ -110,7 +114,7 @@ auth:
 ./cloudeye-exporter -config=clouds.yml
 ```
 
-3.1 出于安全考虑cloudeye-exporter提供了 -s参数, 可以通过命令行交互的方式输入ak sk避免明文配置在clouds.yml文件中引起泄露
+4.1 出于安全考虑cloudeye-exporter提供了 -s参数, 可以通过命令行交互的方式输入ak sk避免明文配置在clouds.yml文件中引起泄露
 ```shell
 ./cloudeye-exporter -s true
 ```

clouds.yml

@@ -2,6 +2,7 @@ global:
   prefix: "huaweicloud"
   scrape_batch_size: 300
   resource_sync_interval_minutes: 180
+  ignore_ssl_verify: false
 auth:
   auth_url: "https://iam.xxx.yyy.com/v3"
   project_name: "{project_name}"

collector/apic.go

@@ -174,6 +174,6 @@ func showDetailsOfInstanceV2(instanceID string) (*model.ShowDetailsOfInstanceV2R
 func getAPICSClient() *apig.ApigClient {
 	return apig.NewApigClient(apig.ApigClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("apig", "v2")).Build())
 }

collector/apig.go

@@ -122,6 +122,6 @@ func getAllAPIGAppsInstances() ([]APIGAppsInfo, error) {
 func getAPIGSClient() *apig.ApigClient {
 	return apig.NewApigClient(apig.ApigClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("apig", "v1.0")).Build())
 }

collector/cbr.go

@@ -89,6 +89,6 @@ func getAllCbrInstancesFromCBR() ([]ResourceBaseInfo, error) {
 func getCBRClient() *cbr.CbrClient {
 	return cbr.NewCbrClient(cbr.CbrClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("cbr", "v3")).Build())
 }

collector/cc.go

@@ -6,6 +6,7 @@ import (
 
 	http_client "github.com/huaweicloud/huaweicloud-sdk-go-v3/core"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config"
 	cc "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3/model"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3/region"
@@ -162,7 +163,8 @@ func getCCClient() *cc.CcClient {
 }
 
 func getCCClientBuilder() *http_client.HcHttpClientBuilder {
-	builder := cc.CcClientBuilder().WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build())
+	builder := cc.CcClientBuilder().WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build()).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify))
 	if endpoint, ok := endpointConfig["cc"]; ok {
 		builder.WithEndpoint(endpoint)
 	} else {

collector/cdm.go

@@ -49,7 +49,7 @@ func (getter CDMInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric
 func getCDMClient() *cdm.CdmClient {
 	return cdm.NewCdmClient(cdm.CdmClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("cdm", "v1.1")).Build())
 }
 

collector/cfw_test.go

@@ -15,7 +15,7 @@ import (
 func TestCFWInfo_GetResourceInfo_configIsNil(t *testing.T) {
 	patches := getPatches()
 	defer patches.Reset()
-	logs.InitLog()
+	logs.InitLog("")
 	cfwInfoTest := CFWInfo{}
 	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
 	assert.Nil(t, labelInfos)
@@ -28,7 +28,7 @@ func TestCFWInfo_GetResourceInfo_dimConfigIsNotExists(t *testing.T) {
 	defer patches.Reset()
 	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
 
-	logs.InitLog()
+	logs.InitLog("")
 	cfwInfoTest := CFWInfo{}
 	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
 	assert.Nil(t, labelInfos)
@@ -43,7 +43,7 @@ func TestCFWInfo_GetResourceInfo_dimConfigIsEmpty(t *testing.T) {
 		"fw_instance_id": nil,
 	}
 	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
-	logs.InitLog()
+	logs.InitLog("")
 	cfwInfoTest := CFWInfo{}
 	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
 	assert.Nil(t, labelInfos)
@@ -60,7 +60,7 @@ func TestCFWInfo_GetResourceInfo_getResourcesFromRMSFailed(t *testing.T) {
 	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
 	patches.ApplyFuncReturn(listResources, nil, errors.New("test err"))
 
-	logs.InitLog()
+	logs.InitLog("")
 	cfwInfoTest := CFWInfo{}
 	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
 	assert.Nil(t, labelInfos)
@@ -76,7 +76,7 @@ func TestCFWInfo_GetResourceInfo_success(t *testing.T) {
 	}
 	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
 	patches.ApplyFuncReturn(listResources, resourceEntityInit(), nil)
-	logs.InitLog()
+	logs.InitLog("")
 	cfwInfoTest := CFWInfo{}
 	// 两个指标，两个资源
 	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()

collector/collector.go

@@ -59,9 +59,13 @@ func (exporter *BaseHuaweiCloudExporter) listMetrics(namespace string) ([]model.
 	allResourcesInfo, metrics := exporter.listAllResources(namespace)
 	logs.Logger.Debugf("[%s] Resource number of %s: %d", exporter.txnKey, namespace, len(allResourcesInfo))
 
-	if len(metrics) > 0 && CloudConf.Global.EpIds != "" {
+	if len(metrics) > 0 {
 		return metrics, allResourcesInfo
 	}
+	// 用户指定了EPID但是metrics为空的场景下,则不应返回数据
+	if CloudConf.Global.EpIds != "" {
+		return nil, nil
+	}
 	logs.Logger.Debugf("[%s] Start to getAllMetric from CES", exporter.txnKey)
 	allMetrics, err := listAllMetrics(namespace)
 	if err != nil {

collector/collector_test.go

@@ -1,15 +1,16 @@
 package collector
 
 import (
-	"github.com/agiledragon/gomonkey/v2"
-	"github.com/huaweicloud/cloudeye-exporter/logs"
-	"go.uber.org/zap/zapcore"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/agiledragon/gomonkey/v2"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/huaweicloud/cloudeye-exporter/logs"
 )
 
 func TestReplaceName(t *testing.T) {
@@ -146,7 +147,7 @@ func TestSetProData1(t *testing.T) {
 		return nil
 	})
 	defer patches.Reset()
-	logs.InitLog()
+	logs.InitLog("")
 	exporter.Collect(nil)
 	exporter.setProData(nil, nil, metricDataArray, resourceInfo, &proMap)
 	assert.Equal(t, 3, len(label.Name))

collector/config.go

@@ -2,6 +2,7 @@ package collector
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"path/filepath"
@@ -21,12 +22,11 @@ type CloudAuth struct {
 	ProjectName string `yaml:"project_name"`
 	ProjectID   string `yaml:"project_id"`
 	DomainName  string `yaml:"domain_name"`
-	AccessKey   string `yaml:"access_key"`
-	Region      string `yaml:"region"`
-	SecretKey   string `yaml:"secret_key"`
-	AuthURL     string `yaml:"auth_url"`
-	UserName    string `yaml:"user_name"`
-	Password    string `yaml:"password"`
+	// 建议您优先使用ReadMe文档中 4.1章节的方式，使用脚本将AccessKey,SecretKey解密后传入，避免在配置文件中明文配置AK SK导致信息泄露
+	AccessKey string `yaml:"access_key"`
+	Region    string `yaml:"region"`
+	SecretKey string `yaml:"secret_key"`
+	AuthURL   string `yaml:"auth_url"`
 }
 
 type Global struct {
@@ -38,6 +38,10 @@ type Global struct {
 	ScrapeBatchSize             int    `yaml:"scrape_batch_size"`
 	ResourceSyncIntervalMinutes int    `yaml:"resource_sync_interval_minutes"`
 	EpIds                       string `yaml:"ep_ids"`
+	MetricsConfPath             string `yaml:"metrics_conf_path"`
+	LogsConfPath                string `yaml:"logs_conf_path"`
+	EndpointsConfPath           string `yaml:"endpoints_conf_path"`
+	IgnoreSSLVerify             bool   `yaml:"ignore_ssl_verify"`
 }
 
 type CloudConfig struct {
@@ -72,8 +76,6 @@ func InitCloudConf(file string) error {
 	if err != nil {
 		return err
 	}
-
-	initEndpointConfig()
 	return err
 }
 
@@ -118,6 +120,18 @@ func SetDefaultConfigValues(config *CloudConfig) {
 	if config.Global.ResourceSyncIntervalMinutes <= 0 {
 		config.Global.ResourceSyncIntervalMinutes = 180
 	}
+
+	if config.Global.MetricsConfPath == "" {
+		config.Global.MetricsConfPath = "./metric.yml"
+	}
+
+	if config.Global.LogsConfPath == "" {
+		config.Global.LogsConfPath = "./logs.yml"
+	}
+
+	if config.Global.EndpointsConfPath == "" {
+		config.Global.EndpointsConfPath = "./endpoints.yml"
+	}
 }
 
 type MetricConf struct {
@@ -129,7 +143,7 @@ var metricConf map[string]MetricConf
 
 func InitMetricConf() error {
 	metricConf = make(map[string]MetricConf)
-	data, err := ioutil.ReadFile("metric.yml")
+	data, err := ioutil.ReadFile(CloudConf.Global.MetricsConfPath)
 	if err != nil {
 		return err
 	}
@@ -158,12 +172,9 @@ type Config struct {
 	DomainName       string
 	EndpointType     string
 	IdentityEndpoint string
-	Password         string
 	Region           string
 	ProjectID        string
 	ProjectName      string
-	Token            string
-	Username         string
 	UserID           string
 }
 
@@ -174,9 +185,8 @@ func InitConfig() error {
 	conf.ProjectName = CloudConf.Auth.ProjectName
 	conf.ProjectID = CloudConf.Auth.ProjectID
 	conf.DomainName = CloudConf.Auth.DomainName
-	conf.Username = CloudConf.Auth.UserName
 	conf.Region = CloudConf.Auth.Region
-	conf.Password = CloudConf.Auth.Password
+	// 安全模式下，ak/sk通过用户交互获取，避免明文方式存在于存储介质中
 	if SecurityMod {
 		conf.AccessKey = TmpAK
 		conf.SecretKey = TmpSK
@@ -186,24 +196,24 @@ func InitConfig() error {
 	}
 
 	if conf.ProjectID == "" && conf.ProjectName == "" {
-		logs.Logger.Error("Init config error: ProjectID or ProjectName must setting.")
+		fmt.Printf("Init config error: ProjectID or ProjectName must setting.")
 		return errors.New("init config error: ProjectID or ProjectName must setting")
 	}
 	req, err := http.NewRequest("GET", conf.IdentityEndpoint, nil)
 	if err != nil {
-		logs.Logger.Error("Auth url is invalid.")
+		fmt.Printf("Auth url is invalid.")
 		return err
 	}
 	host = req.Host
 
 	if conf.ProjectID == "" {
 		resp, err := getProjectInfo()
 		if err != nil {
-			logs.Logger.Errorf("Get project info error: %s", err.Error())
+			fmt.Printf("Get project info error: %s", err.Error())
 			return err
 		}
 		if len(*resp.Projects) == 0 {
-			logs.Logger.Error("project info is empty")
+			fmt.Printf("Project info is empty")
 			return errors.New("project info is empty")
 		}
 
@@ -223,18 +233,23 @@ func getProjectInfo() (*model.KeystoneListProjectsResponse, error) {
 					WithAk(conf.AccessKey).
 					WithSk(conf.SecretKey).
 					Build()).
-			WithHttpConfig(config.DefaultHttpConfig().
-				WithIgnoreSSLVerification(true)).
+			WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 			Build())
 	return iamclient.KeystoneListProjects(&model.KeystoneListProjectsRequest{Name: &conf.ProjectName})
 }
 
 var endpointConfig map[string]string
 
-func initEndpointConfig() {
-	context, err := ioutil.ReadFile("endpoints.yml")
+func InitEndpointConfig(path string) {
+	realPath, err := NormalizePath(path)
 	if err != nil {
-		logs.Logger.Errorf("Init endpoint config error: %s", err.Error())
+		logs.Logger.Errorf("Normalize endpoint config err: %s", err.Error())
+		return
+	}
+
+	context, err := ioutil.ReadFile(realPath)
+	if err != nil {
+		logs.Logger.Infof("Invalid endpoint config path, default config will be used instead")
 		return
 	}
 	err = yaml.Unmarshal(context, &endpointConfig)

collector/config_test.go

@@ -109,7 +109,7 @@ func TestInitEndpointConfig(t *testing.T) {
 			patches := testCase.patches()
 			patches.ApplyMethod(&logs.Logger, "Errorf", func(logger *logs.LoggerConstructor, template string, args ...interface{}) {})
 			defer patches.Reset()
-			initEndpointConfig()
+			InitEndpointConfig("")
 			testCase.expect(t)
 		})
 	}

collector/ddms.go

@@ -114,6 +114,6 @@ func getDdmsInstanceNodes(instanceId string) ([]ddmmodel.ShowNodeResponse, error
 func getDDMSClient() *ddm.DdmClient {
 	return ddm.NewDdmClient(ddm.DdmClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("ddm", "v1")).Build())
 }

collector/dds.go

@@ -125,6 +125,6 @@ func fmtDdsInstance(instance ddsmodel.QueryInstanceResponse) DdsInstanceInfo {
 func getDDSClient() *dds.DdsClient {
 	return dds.NewDdsClient(dds.DdsClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(CloudConf.Global.IgnoreSSLVerify)).
 		WithEndpoint(getEndpoint("dds", "v3")).Build())
 }