feat: adding support to cgroup level io.weight setting

Signed-off-by: Jianyu Sun <sjycsfldf@gmail.com>
Signed-off-by: Robin Lu <robin.lu@bytedance.com>

cmd/katalyst-agent/app/options/qrm/io_plugin.go

@@ -38,8 +38,9 @@ type WritebackThrottlingOption struct {
 }
 
 type IOWeightOption struct {
-	EnableSettingIOWeight      bool
-	IOWeightQoSLevelConfigFile string
+	EnableSettingIOWeight         bool
+	IOWeightQoSLevelConfigFile    string
+	IOWeightCgroupLevelConfigFile string
 }
 
 func NewIOOptions() *IOOptions {
@@ -51,8 +52,9 @@ func NewIOOptions() *IOOptions {
 			WBTValueSSD:      2000,
 		},
 		IOWeightOption: IOWeightOption{
-			EnableSettingIOWeight:      false,
-			IOWeightQoSLevelConfigFile: "",
+			EnableSettingIOWeight:         false,
+			IOWeightQoSLevelConfigFile:    "",
+			IOWeightCgroupLevelConfigFile: "",
 		},
 	}
 }
@@ -72,6 +74,8 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.EnableSettingIOWeight, "if set it to true, io.weight related control operations will be executed")
 	fs.StringVar(&o.IOWeightQoSLevelConfigFile, "io-weight-qos-config-file",
 		o.IOWeightQoSLevelConfigFile, "the absolute path of io.weight qos config file")
+	fs.StringVar(&o.IOWeightCgroupLevelConfigFile, "io-weight-cgroup-config-file",
+		o.IOWeightCgroupLevelConfigFile, "the absolute path of io.weight cgroup config file")
 }
 
 func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error {
@@ -81,5 +85,6 @@ func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error {
 	conf.WBTValueSSD = o.WBTValueSSD
 	conf.EnableSettingIOWeight = o.EnableSettingIOWeight
 	conf.IOWeightQoSLevelConfigFile = o.IOWeightQoSLevelConfigFile
+	conf.IOWeightCgroupLevelConfigFile = o.IOWeightCgroupLevelConfigFile
 	return nil
 }

pkg/agent/qrm-plugins/io/handlers/ioweight/const.go

@@ -23,4 +23,5 @@ const (
 
 	controlKnobKeyIOWeight = "io_weight"
 	cgroupIOWeightName     = "io.weight"
+	defaultDevID           = "default"
 )

pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go

@@ -21,6 +21,7 @@ package ioweight
 
 import (
 	"context"
+	"strconv"
 
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/commonstate"
 	coreconfig "github.com/kubewharf/katalyst-core/pkg/config"
@@ -33,6 +34,36 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/util/native"
 )
 
+func applyIOWeightCgroupLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter) {
+	if conf.IOWeightCgroupLevelConfigFile == "" {
+		general.Errorf("IOWeightCgroupLevelConfigFile isn't configured")
+		return
+	}
+
+	ioWightCgroupLevelConfigs := make(map[string]uint64)
+	err := general.LoadJsonConfig(conf.IOWeightCgroupLevelConfigFile, &ioWightCgroupLevelConfigs)
+	if err != nil {
+		general.Errorf("load IOWeightCgroupLevelConfig failed with error: %v", err)
+		return
+	}
+
+	for relativeCgPath, weight := range ioWightCgroupLevelConfigs {
+		err := cgroupmgr.ApplyIOWeightWithRelativePath(relativeCgPath, defaultDevID, weight)
+		if err != nil {
+			general.Errorf("ApplyIOWeightWithRelativePath for devID: %s in relativeCgPath: %s failed with error: %v",
+				defaultDevID, relativeCgPath, err)
+		} else {
+			general.Infof("ApplyIOWeightWithRelativePath for devID: %s, weight: %d in relativeCgPath: %s successfully",
+				defaultDevID, weight, relativeCgPath)
+			_ = emitter.StoreInt64(metricNameIOWeight, int64(weight), metrics.MetricTypeNameRaw,
+				metrics.ConvertMapToTags(map[string]string{
+					"cgPath": relativeCgPath,
+				})...)
+
+		}
+	}
+}
+
 func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration,
 	emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) {
 	if conf.IOWeightQoSLevelConfigFile == "" {
@@ -57,6 +88,9 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration,
 			general.Warningf("get nil pod from metaServer")
 			continue
 		}
+		if conf.QoSConfiguration == nil {
+			continue
+		}
 		qosConfig := conf.QoSConfiguration
 		qosLevel, err := qosConfig.GetQoSLevelForPod(pod)
 		if err != nil {
@@ -65,17 +99,27 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration,
 		}
 		qosLevelDefaultValue, ok := extraControlKnobConfigs[controlKnobKeyIOWeight].QoSLevelToDefaultValue[qosLevel]
 		if !ok {
-			general.Warningf("no QoSLevelToDefaultValue in extraControlKnobConfigs")
 			continue
 		}
-
 		for _, containerStatus := range pod.Status.ContainerStatuses {
 			podUID, containerID := string(pod.UID), native.TrimContainerIDPrefix(containerStatus.ContainerID)
 			err := cgroupmgr.ApplyUnifiedDataForContainer(podUID, containerID, extraControlKnobConfigs[controlKnobKeyIOWeight].CgroupSubsysName, cgroupIOWeightName, qosLevelDefaultValue)
 			if err != nil {
 				general.Warningf("ApplyUnifiedDataForContainer failed:%v", err)
 				continue
 			}
+
+			ioWeightValue, err := strconv.ParseInt(qosLevelDefaultValue, 10, 64)
+			if err != nil {
+				general.Warningf("strconv.ParseInt failed, string=%v, err=%v", qosLevelDefaultValue, err)
+				continue
+			}
+
+			_ = emitter.StoreInt64(metricNameIOWeight, ioWeightValue, metrics.MetricTypeNameRaw,
+				metrics.ConvertMapToTags(map[string]string{
+					"podUID":      podUID,
+					"containerID": containerID,
+				})...)
 		}
 	}
 }
@@ -111,4 +155,9 @@ func IOWeightTaskFunc(conf *coreconfig.Configuration,
 	if len(conf.IOWeightQoSLevelConfigFile) > 0 {
 		applyIOWeightQoSLevelConfig(conf, emitter, metaServer)
 	}
+
+	// checking cgroup-level io.weight configuration.
+	if len(conf.IOWeightCgroupLevelConfigFile) > 0 {
+		applyIOWeightCgroupLevelConfig(conf, emitter)
+	}
 }