diff --git a/cmd/katalyst-agent/app/options/qrm/io_plugin.go b/cmd/katalyst-agent/app/options/qrm/io_plugin.go index 7e6bd67734..75ee2402f0 100644 --- a/cmd/katalyst-agent/app/options/qrm/io_plugin.go +++ b/cmd/katalyst-agent/app/options/qrm/io_plugin.go @@ -38,8 +38,9 @@ type WritebackThrottlingOption struct { } type IOWeightOption struct { - EnableSettingIOWeight bool - IOWeightQoSLevelConfigFile string + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string + IOWeightCgroupLevelConfigFile string } func NewIOOptions() *IOOptions { @@ -51,8 +52,9 @@ func NewIOOptions() *IOOptions { WBTValueSSD: 2000, }, IOWeightOption: IOWeightOption{ - EnableSettingIOWeight: false, - IOWeightQoSLevelConfigFile: "", + EnableSettingIOWeight: false, + IOWeightQoSLevelConfigFile: "", + IOWeightCgroupLevelConfigFile: "", }, } } @@ -72,6 +74,8 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.EnableSettingIOWeight, "if set it to true, io.weight related control operations will be executed") fs.StringVar(&o.IOWeightQoSLevelConfigFile, "io-weight-qos-config-file", o.IOWeightQoSLevelConfigFile, "the absolute path of io.weight qos config file") + fs.StringVar(&o.IOWeightCgroupLevelConfigFile, "io-weight-cgroup-config-file", + o.IOWeightCgroupLevelConfigFile, "the absolute path of io.weight cgroup config file") } func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { @@ -81,5 +85,6 @@ func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { conf.WBTValueSSD = o.WBTValueSSD conf.EnableSettingIOWeight = o.EnableSettingIOWeight conf.IOWeightQoSLevelConfigFile = o.IOWeightQoSLevelConfigFile + conf.IOWeightCgroupLevelConfigFile = o.IOWeightCgroupLevelConfigFile return nil } diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go index a73498ba31..0f80ea7617 100644 --- a/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go @@ -23,4 +23,5 @@ const ( controlKnobKeyIOWeight = "io_weight" cgroupIOWeightName = "io.weight" + defaultDevID = "default" ) diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go index faccd408fa..52a9312b19 100644 --- a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go @@ -21,6 +21,7 @@ package ioweight import ( "context" + "strconv" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/commonstate" coreconfig "github.com/kubewharf/katalyst-core/pkg/config" @@ -33,6 +34,36 @@ import ( "github.com/kubewharf/katalyst-core/pkg/util/native" ) +func applyIOWeightCgroupLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter) { + if conf.IOWeightCgroupLevelConfigFile == "" { + general.Errorf("IOWeightCgroupLevelConfigFile isn't configured") + return + } + + ioWightCgroupLevelConfigs := make(map[string]uint64) + err := general.LoadJsonConfig(conf.IOWeightCgroupLevelConfigFile, &ioWightCgroupLevelConfigs) + if err != nil { + general.Errorf("load IOWeightCgroupLevelConfig failed with error: %v", err) + return + } + + for relativeCgPath, weight := range ioWightCgroupLevelConfigs { + err := cgroupmgr.ApplyIOWeightWithRelativePath(relativeCgPath, defaultDevID, weight) + if err != nil { + general.Errorf("ApplyIOWeightWithRelativePath for devID: %s in relativeCgPath: %s failed with error: %v", + defaultDevID, relativeCgPath, err) + } else { + general.Infof("ApplyIOWeightWithRelativePath for devID: %s, weight: %d in relativeCgPath: %s successfully", + defaultDevID, weight, relativeCgPath) + _ = emitter.StoreInt64(metricNameIOWeight, int64(weight), metrics.MetricTypeNameRaw, + metrics.ConvertMapToTags(map[string]string{ + "cgPath": relativeCgPath, + })...) + + } + } +} + func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) { if conf.IOWeightQoSLevelConfigFile == "" { @@ -57,6 +88,9 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, general.Warningf("get nil pod from metaServer") continue } + if conf.QoSConfiguration == nil { + continue + } qosConfig := conf.QoSConfiguration qosLevel, err := qosConfig.GetQoSLevelForPod(pod) if err != nil { @@ -65,10 +99,8 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, } qosLevelDefaultValue, ok := extraControlKnobConfigs[controlKnobKeyIOWeight].QoSLevelToDefaultValue[qosLevel] if !ok { - general.Warningf("no QoSLevelToDefaultValue in extraControlKnobConfigs") continue } - for _, containerStatus := range pod.Status.ContainerStatuses { podUID, containerID := string(pod.UID), native.TrimContainerIDPrefix(containerStatus.ContainerID) err := cgroupmgr.ApplyUnifiedDataForContainer(podUID, containerID, extraControlKnobConfigs[controlKnobKeyIOWeight].CgroupSubsysName, cgroupIOWeightName, qosLevelDefaultValue) @@ -76,6 +108,18 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, general.Warningf("ApplyUnifiedDataForContainer failed:%v", err) continue } + + ioWeightValue, err := strconv.ParseInt(qosLevelDefaultValue, 10, 64) + if err != nil { + general.Warningf("strconv.ParseInt failed, string=%v, err=%v", qosLevelDefaultValue, err) + continue + } + + _ = emitter.StoreInt64(metricNameIOWeight, ioWeightValue, metrics.MetricTypeNameRaw, + metrics.ConvertMapToTags(map[string]string{ + "podUID": podUID, + "containerID": containerID, + })...) } } } @@ -111,4 +155,9 @@ func IOWeightTaskFunc(conf *coreconfig.Configuration, if len(conf.IOWeightQoSLevelConfigFile) > 0 { applyIOWeightQoSLevelConfig(conf, emitter, metaServer) } + + // checking cgroup-level io.weight configuration. + if len(conf.IOWeightCgroupLevelConfigFile) > 0 { + applyIOWeightCgroupLevelConfig(conf, emitter) + } } diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go new file mode 100644 index 0000000000..318f236a3c --- /dev/null +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go @@ -0,0 +1,368 @@ +//go:build linux +// +build linux + +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ioweight + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + coreconfig "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/config/agent" + configagent "github.com/kubewharf/katalyst-core/pkg/config/agent" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" + "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm" + "github.com/kubewharf/katalyst-core/pkg/config/generic" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + metaagent "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" + "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric" + "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" + "github.com/kubewharf/katalyst-core/pkg/metrics" + "github.com/kubewharf/katalyst-core/pkg/util/machine" +) + +func makeMetaServer() (*metaserver.MetaServer, error) { + server := &metaserver.MetaServer{ + MetaAgent: &metaagent.MetaAgent{}, + } + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 1, 2) + if err != nil { + return nil, err + } + + server.KatalystMachineInfo = &machine.KatalystMachineInfo{ + CPUTopology: cpuTopology, + } + server.MetricsFetcher = metric.NewFakeMetricsFetcher(metrics.DummyMetrics{}) + return server, nil +} + +func TestIOWeightTaskFunc(t *testing.T) { + t.Parallel() + + IOWeightTaskFunc(nil, + nil, &dynamicconfig.DynamicAgentConfiguration{}, nil, nil) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: false, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, nil, nil) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, nil) + + metaServer, err := makeMetaServer() + assert.NoError(t, err) + metaServer.PodFetcher = &pod.PodFetcherStub{PodList: []*v1.Pod{}} + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + normalPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "normalPod", + Name: "normalPod", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c", + }, + }, + }, + } + + metaServer.PodFetcher = &pod.PodFetcherStub{PodList: []*v1.Pod{normalPod}} + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: false, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: "fake", + IOWeightCgroupLevelConfigFile: "fake", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + applyIOWeightCgroupLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightCgroupLevelConfigFile: "", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}) + + applyIOWeightCgroupLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightCgroupLevelConfigFile: "fake", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}) + + applyIOWeightQoSLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: "", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, metaServer) + + applyIOWeightQoSLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: "fake", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, metaServer) + + metaServerEmpty, err := makeMetaServer() + assert.NoError(t, err) + metaServerEmpty.PodFetcher = &pod.PodFetcherStub{PodList: []*v1.Pod{}} + + jsonContent := `{ + "io_weight": { + "control_knob_info": { + "cgroup_subsys_name": "io", + "cgroup_version_to_iface_name": { + "v1": "", + "v2": "io.weight" + }, + "control_knob_value": "100", + "oci_property_name": "" + }, + "pod_explicitly_annotation_key": "IOWeightValue", + "qos_level_to_default_value": { + "dedicated_cores": "500", + "shared_cores": "500" + } + } + }` + + // Create a temporary file + tempFile, err := ioutil.TempFile("", "test.json") + if err != nil { + fmt.Println("Error creating temporary file:", err) + return + } + defer os.Remove(tempFile.Name()) // Defer removing the temporary file + + // Write the JSON content to the temporary file + if _, err := tempFile.WriteString(jsonContent); err != nil { + fmt.Println("Error writing to temporary file:", err) + return + } + + absPath, err := filepath.Abs(tempFile.Name()) + if err != nil { + fmt.Println("Error obtaining absolute path:", err) + return + } + + jsonContent2 := `{ + "fake": 200, + "fake2": 300 + }` + + // Create a temporary file + tempFile2, err := ioutil.TempFile("", "test2.json") + if err != nil { + fmt.Println("Error creating temporary file:", err) + return + } + defer os.Remove(tempFile2.Name()) // Defer removing the temporary file + + // Write the JSON content to the temporary file + if _, err := tempFile2.WriteString(jsonContent2); err != nil { + fmt.Println("Error writing to temporary file:", err) + return + } + + absPathCgroup, err := filepath.Abs(tempFile2.Name()) + if err != nil { + fmt.Println("Error obtaining absolute path:", err) + return + } + + applyIOWeightCgroupLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightCgroupLevelConfigFile: absPathCgroup, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}) + + applyIOWeightQoSLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: absPath, + }, + }, + }, + }, + }, + GenericConfiguration: &generic.GenericConfiguration{ + QoSConfiguration: nil, + }, + }, metrics.DummyMetrics{}, metaServer) + + applyIOWeightQoSLevelConfig(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: absPath, + }, + }, + }, + }, + }, + GenericConfiguration: &generic.GenericConfiguration{ + QoSConfiguration: nil, + }, + }, metrics.DummyMetrics{}, metaServerEmpty) +} diff --git a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go index 4d3aace918..f637e6e010 100644 --- a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go +++ b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go @@ -122,7 +122,7 @@ func (p *StaticPolicy) Start() (err error) { } if p.enableSettingWBT { general.Infof("setWBT enabled") - err := periodicalhandler.RegisterPeriodicalHandler(qrm.QRMMemoryPluginPeriodicalHandlerGroupName, + err := periodicalhandler.RegisterPeriodicalHandler(qrm.QRMIOPluginPeriodicalHandlerGroupName, dirtymem.EnableSetDirtyMemPeriodicalHandlerName, dirtymem.SetDirtyMem, 300*time.Second) if err != nil { general.Infof("setSockMem failed, err=%v", err) diff --git a/pkg/config/agent/qrm/io_plugin.go b/pkg/config/agent/qrm/io_plugin.go index 1184431e70..c74329ff02 100644 --- a/pkg/config/agent/qrm/io_plugin.go +++ b/pkg/config/agent/qrm/io_plugin.go @@ -31,8 +31,9 @@ type WritebackThrottlingOption struct { } type IOWeightOption struct { - EnableSettingIOWeight bool - IOWeightQoSLevelConfigFile string + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string + IOWeightCgroupLevelConfigFile string } func NewIOQRMPluginConfig() *IOQRMPluginConfig {