Merge pull request #59 from cxl123156/br_release_sdk_v3

发布2.0.7版本

README_cn.md

@@ -50,6 +50,7 @@ Prometheus是用于展示大型测量数据的开源可视化工具，在工业
 |MapReduce服务|SYS.MRS |√|RMS/云服务|
 |湖仓构建服务|SYS.LakeFormation |√|RMS/云服务|
 |智能数据湖运营平台|SYS.DAYU |√|云服务|
+|云防火墙|SYS.CFW |√|RMS|
 
 注：自定义标签时，key只能包含大写字母、小写字母以及中划线
 

collector/cfw.go

@@ -0,0 +1,62 @@
+package collector
+
+import (
+	"time"
+
+	cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
+
+	"github.com/huaweicloud/cloudeye-exporter/logs"
+)
+
+var cfwServerInfo serversInfo
+
+type CFWInfo struct{}
+
+func (cfw CFWInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricInfoList) {
+	resourceInfos := map[string]labelInfo{}
+	filterMetrics := make([]cesmodel.MetricInfoList, 0)
+	cfwServerInfo.Lock()
+	defer cfwServerInfo.Unlock()
+	if cfwServerInfo.LabelInfo == nil || time.Now().Unix() > cfwServerInfo.TTL {
+		cfwConfigMap := getMetricConfigMap("SYS.CFW")
+		if cfwConfigMap == nil {
+			logs.Logger.Warn("Metric config is nil.")
+			return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics
+		}
+
+		if _, ok := cfwConfigMap["fw_instance_id"]; !ok {
+			logs.Logger.Warn("Metric config is nil of SYS.CFW of fw_instance_id.")
+			return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics
+		}
+
+		metricNames := cfwConfigMap["fw_instance_id"]
+		if len(metricNames) == 0 {
+			logs.Logger.Warn("Metric config is empty of SYS.CFW of fw_instance_id.")
+			return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics
+		}
+
+		servers, err := getResourcesBaseInfoFromRMS("cfw", "cfw_instance")
+		if err != nil {
+			logs.Logger.Errorf("Get resource base info from RMS Server error:", err.Error())
+			return ecsInfo.LabelInfo, ecsInfo.FilterMetrics
+		}
+
+		for _, server := range servers {
+			metrics := buildSingleDimensionMetrics(metricNames, "SYS.CFW", "fw_instance_id", server.ID)
+			filterMetrics = append(filterMetrics, metrics...)
+			info := labelInfo{
+				Name:  []string{"name", "ep_id"},
+				Value: []string{server.Name, server.EpId},
+			}
+			keys, values := getTags(server.Tags)
+			info.Name = append(info.Name, keys...)
+			info.Value = append(info.Value, values...)
+			resourceInfos[GetResourceKeyFromMetricInfo(metrics[0])] = info
+		}
+
+		cfwServerInfo.LabelInfo = resourceInfos
+		cfwServerInfo.FilterMetrics = filterMetrics
+		cfwServerInfo.TTL = time.Now().Add(TTL).Unix()
+	}
+	return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics
+}

collector/cfw_test.go

@@ -0,0 +1,180 @@
+package collector
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/huaweicloud/cloudeye-exporter/logs"
+)
+
+func TestCFWInfo_GetResourceInfo_configIsNil(t *testing.T) {
+	patches := getPatches()
+	defer patches.Reset()
+	logs.InitLog()
+	cfwInfoTest := CFWInfo{}
+	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
+	assert.Nil(t, labelInfos)
+	assert.Nil(t, filterMetrics)
+}
+
+func TestCFWInfo_GetResourceInfo_dimConfigIsNotExists(t *testing.T) {
+	metricConfigMap := map[string][]string{}
+	patches := getPatches()
+	defer patches.Reset()
+	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
+
+	logs.InitLog()
+	cfwInfoTest := CFWInfo{}
+	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
+	assert.Nil(t, labelInfos)
+	assert.Nil(t, filterMetrics)
+}
+
+func TestCFWInfo_GetResourceInfo_dimConfigIsEmpty(t *testing.T) {
+	patches := getPatches()
+	defer patches.Reset()
+
+	metricConfigMap := map[string][]string{
+		"fw_instance_id": nil,
+	}
+	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
+	logs.InitLog()
+	cfwInfoTest := CFWInfo{}
+	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
+	assert.Nil(t, labelInfos)
+	assert.Nil(t, filterMetrics)
+}
+
+func TestCFWInfo_GetResourceInfo_getResourcesFromRMSFailed(t *testing.T) {
+	patches := getPatches()
+	defer patches.Reset()
+
+	metricConfigMap := map[string][]string{
+		"fw_instance_id": {"metric1", "metric2"},
+	}
+	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
+	patches.ApplyFuncReturn(listResources, nil, errors.New("test err"))
+
+	logs.InitLog()
+	cfwInfoTest := CFWInfo{}
+	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
+	assert.Nil(t, labelInfos)
+	assert.Nil(t, filterMetrics)
+}
+
+func TestCFWInfo_GetResourceInfo_success(t *testing.T) {
+	patches := getPatches()
+	defer patches.Reset()
+
+	metricConfigMap := map[string][]string{
+		"fw_instance_id": {"metric1", "metric2"},
+	}
+	patches.ApplyFuncReturn(getMetricConfigMap, metricConfigMap)
+	patches.ApplyFuncReturn(listResources, resourceEntityInit(), nil)
+	logs.InitLog()
+	cfwInfoTest := CFWInfo{}
+	// 两个指标，两个资源
+	labelInfos, filterMetrics := cfwInfoTest.GetResourceInfo()
+	assert.NotNil(t, labelInfos)
+	assert.Equal(t, 2, len(labelInfos))
+	assert.NotNil(t, filterMetrics)
+	assert.Equal(t, 4, len(filterMetrics))
+}
+
+func getPatches() *gomonkey.Patches {
+	confLoader := &logs.ConfLoader{}
+	patches := gomonkey.ApplyMethodFunc(*confLoader, "LoadFile", func(fPath string, cfg interface{}) error {
+		cfgTmp, _ := cfg.(*map[string][]logs.Config)
+		cfgPointer := make(map[string][]logs.Config)
+		cfgPointer["business"] = []logs.Config{
+			{
+				Level: zapcore.InfoLevel,
+			},
+		}
+		*cfgTmp = cfgPointer
+		return nil
+	})
+	return patches
+}
+
+func resourceEntityInit() []model.ResourceEntity {
+	id1 := "xxxx2"
+	name1 := "test"
+	epId1 := "1"
+	epName1 := "测试企业1"
+	checksum1 := "xxxb"
+	create1 := "2023-12-23T07:58:14.000Z"
+	update1 := "2023-12-23T07:58:14.000Z"
+	provisioningState1 := "Succeeded"
+
+	id2 := "xxxx3"
+	name2 := "test2"
+	epId2 := "2"
+	epName2 := "测试企业2"
+	checksum2 := "xxxc"
+	create2 := "2023-12-23T07:58:14.000Z"
+	update2 := "2023-12-25T07:58:14.000Z"
+	provisioningState2 := "Succeeded"
+
+	provider := "cfw"
+	typestr := "cfw_instance"
+	regionId := "cn-north-7"
+	projectId := "xxxx0"
+	projectName := "cn-north-7"
+	tags := map[string]string{}
+	properties := map[string]interface{}{
+		"domain_id":             "xxxx001",
+		"engine_type":           0,
+		"enterprise_project_id": "1",
+		"service_type":          0,
+		"project_id":            "xxxx002",
+		"fw_instance_name":      "test",
+		"name":                  "1703318294687",
+		"policy_count":          0,
+		"fw_instance_id":        "xxxx003",
+		"status":                "status",
+	}
+
+	response := []model.ResourceEntity{
+		{
+			Id:                &id1,
+			Name:              &name1,
+			Provider:          &provider,
+			Type:              &typestr,
+			RegionId:          &regionId,
+			ProjectId:         &projectId,
+			ProjectName:       &projectName,
+			EpId:              &epId1,
+			EpName:            &epName1,
+			Checksum:          &checksum1,
+			Created:           &create1,
+			Updated:           &update1,
+			ProvisioningState: &provisioningState1,
+			Tags:              tags,
+			Properties:        properties,
+		},
+		{
+			Id:                &id2,
+			Name:              &name2,
+			Provider:          &provider,
+			Type:              &typestr,
+			RegionId:          &regionId,
+			ProjectId:         &projectId,
+			ProjectName:       &projectName,
+			EpId:              &epId2,
+			EpName:            &epName2,
+			Checksum:          &checksum2,
+			Created:           &create2,
+			Updated:           &update2,
+			ProvisioningState: &provisioningState2,
+			Tags:              tags,
+			Properties:        properties,
+		},
+	}
+	return response
+}

collector/collector.go

@@ -45,6 +45,11 @@ func GetMonitoringCollector(namespaces []string) *BaseHuaweiCloudExporter {
 	return exporter
 }
 
+type PrometheusMetricMap = struct {
+	sync.RWMutex
+	MetricMap map[string]bool // key:txnKey value: metric map for deduplicate label key:label
+}
+
 // Describe simply sends the two Descs in the struct to the channel.
 func (exporter *BaseHuaweiCloudExporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- prometheus.NewDesc("dummy", "dummy", nil, nil)
@@ -68,7 +73,7 @@ func (exporter *BaseHuaweiCloudExporter) listMetrics(namespace string) ([]model.
 }
 
 func (exporter *BaseHuaweiCloudExporter) setProData(ctx context.Context, ch chan<- prometheus.Metric,
-	dataList []model.BatchMetricData, allResourcesInfo map[string]labelInfo) {
+	dataList []model.BatchMetricData, allResourcesInfo map[string]labelInfo, metricMap *PrometheusMetricMap) {
 	defer func() {
 		if err := recover(); err != nil {
 			logs.Logger.Errorf("[%s] SetProData error: %+v", exporter.txnKey, err)
@@ -93,12 +98,25 @@ func (exporter *BaseHuaweiCloudExporter) setProData(ctx context.Context, ch chan
 				exporter.txnKey, err.Error(), fqName, label)
 			continue
 		}
+		dimArray := make([]string, 0, len(*metric.Dimensions))
+		for _, dimension := range *metric.Dimensions {
+			dimArray = append(dimArray, dimension.Name)
+		}
+		dimNameStr := strings.Join(dimArray, ",")
+		if isAgentMetric(*metric.Namespace) && isMetricLabelConflict(fqName, label, metricMap) {
+			logs.Logger.Warnf("[%s] Metric label conflict, namespace: %s , dimension: %s, metric name: %s", exporter.txnKey, *metric.Namespace, dimNameStr, metric.MetricName)
+			continue
+		}
 		if err := sendMetricData(ctx, ch, proMetric); err != nil {
 			logs.Logger.Errorf("[%s] Context has canceled, no need to send metric data, metric name: %s", exporter.txnKey, fqName)
 		}
 	}
 }
 
+func isAgentMetric(namespace string) bool {
+	return namespace == "AGT.ECS" || namespace == "SERVICE.BMS"
+}
+
 func getLabel(metric model.BatchMetricData, info map[string]labelInfo) labelInfo {
 	label := getDimLabel(metric)
 	if extendLabel, exist := info[GetResourceKeyFromMetricData(metric)]; exist {
@@ -140,7 +158,7 @@ func isContainsInStringArr(target string, array []string) bool {
 	return false
 }
 
-func (exporter *BaseHuaweiCloudExporter) collectMetricByNamespace(ctx context.Context, ch chan<- prometheus.Metric, namespace string) {
+func (exporter *BaseHuaweiCloudExporter) collectMetricByNamespace(ctx context.Context, ch chan<- prometheus.Metric, namespace string, proMap *PrometheusMetricMap) {
 	defer func() {
 		if err := recover(); err != nil {
 			logs.Logger.Errorf("[%s] recover error: %+v", exporter.txnKey, err)
@@ -181,7 +199,7 @@ func (exporter *BaseHuaweiCloudExporter) collectMetricByNamespace(ctx context.Co
 				if err != nil {
 					return
 				}
-				exporter.setProData(ctx, ch, *dataList, allResourcesInfo)
+				exporter.setProData(ctx, ch, *dataList, allResourcesInfo, proMap)
 			}(tmpMetrics)
 			tmpMetrics = make([]model.MetricInfo, 0, exporter.ScrapeBatchSize)
 		}
@@ -215,11 +233,15 @@ func (exporter *BaseHuaweiCloudExporter) Collect(ch chan<- prometheus.Metric) {
 
 	logs.Logger.Debugf("[%s] Start to collect data", exporter.txnKey)
 	var wg sync.WaitGroup
+	proMap := PrometheusMetricMap{
+		RWMutex:   sync.RWMutex{},
+		MetricMap: make(map[string]bool),
+	}
 	for _, namespace := range exporter.Namespaces {
 		wg.Add(1)
 		go func(ctx context.Context, ch chan<- prometheus.Metric, namespace string) {
 			defer wg.Done()
-			exporter.collectMetricByNamespace(ctx, ch, namespace)
+			exporter.collectMetricByNamespace(ctx, ch, namespace, &proMap)
 		}(ctx, ch, namespace)
 	}
 	wg.Wait()
@@ -256,3 +278,24 @@ func getLatestData(data []model.DatapointForBatchMetric) (float64, error) {
 
 	return *data[len(data)-1].Average, nil
 }
+
+func isMetricLabelConflict(fqName string, label labelInfo, metricMap *PrometheusMetricMap) bool {
+	labelArray := make([]string, 0, len(label.Name))
+	for i := range label.Name {
+		labelTmp := fmt.Sprintf("%s=%s", label.Name[i], label.Value[i])
+		labelArray = append(labelArray, labelTmp)
+	}
+	labelResult := fmt.Sprintf("%s{%s}", fqName, strings.Join(labelArray, ","))
+
+	metricMap.RLock()
+	_, txnMapOk := metricMap.MetricMap[labelResult]
+	metricMap.RUnlock()
+	if txnMapOk {
+		return true
+	} else {
+		metricMap.Lock()
+		metricMap.MetricMap[labelResult] = true
+		metricMap.Unlock()
+	}
+	return false
+}

collector/collector_test.go

@@ -1,9 +1,14 @@
 package collector
 
 import (
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/huaweicloud/cloudeye-exporter/logs"
+	"go.uber.org/zap/zapcore"
+
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
 	"github.com/stretchr/testify/assert"
 )

collector/extensions.go

@@ -50,6 +50,7 @@ var (
 		"SYS.LakeFormation": LakeFormationInfo{},
 		"SYS.MRS":           MRSInfo{},
 		"SYS.DAYU":          DayuInfo{},
+		"SYS.CFW":           CFWInfo{},
 	}
 )