diff --git a/cmd/katalyst-agent/app/options/qrm/io_plugin.go b/cmd/katalyst-agent/app/options/qrm/io_plugin.go index 7e6bd67734..75ee2402f0 100644 --- a/cmd/katalyst-agent/app/options/qrm/io_plugin.go +++ b/cmd/katalyst-agent/app/options/qrm/io_plugin.go @@ -38,8 +38,9 @@ type WritebackThrottlingOption struct { } type IOWeightOption struct { - EnableSettingIOWeight bool - IOWeightQoSLevelConfigFile string + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string + IOWeightCgroupLevelConfigFile string } func NewIOOptions() *IOOptions { @@ -51,8 +52,9 @@ func NewIOOptions() *IOOptions { WBTValueSSD: 2000, }, IOWeightOption: IOWeightOption{ - EnableSettingIOWeight: false, - IOWeightQoSLevelConfigFile: "", + EnableSettingIOWeight: false, + IOWeightQoSLevelConfigFile: "", + IOWeightCgroupLevelConfigFile: "", }, } } @@ -72,6 +74,8 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.EnableSettingIOWeight, "if set it to true, io.weight related control operations will be executed") fs.StringVar(&o.IOWeightQoSLevelConfigFile, "io-weight-qos-config-file", o.IOWeightQoSLevelConfigFile, "the absolute path of io.weight qos config file") + fs.StringVar(&o.IOWeightCgroupLevelConfigFile, "io-weight-cgroup-config-file", + o.IOWeightCgroupLevelConfigFile, "the absolute path of io.weight cgroup config file") } func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { @@ -81,5 +85,6 @@ func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { conf.WBTValueSSD = o.WBTValueSSD conf.EnableSettingIOWeight = o.EnableSettingIOWeight conf.IOWeightQoSLevelConfigFile = o.IOWeightQoSLevelConfigFile + conf.IOWeightCgroupLevelConfigFile = o.IOWeightCgroupLevelConfigFile return nil } diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go index a73498ba31..0f80ea7617 100644 --- a/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go @@ -23,4 +23,5 @@ const ( controlKnobKeyIOWeight = "io_weight" cgroupIOWeightName = "io.weight" + defaultDevID = "default" ) diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go index faccd408fa..2297a2bd0c 100644 --- a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go @@ -21,6 +21,7 @@ package ioweight import ( "context" + "sync" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/commonstate" coreconfig "github.com/kubewharf/katalyst-core/pkg/config" @@ -33,6 +34,35 @@ import ( "github.com/kubewharf/katalyst-core/pkg/util/native" ) +var ( + initializeOnce sync.Once +) + +func applyIOWeightCgroupLevelConfig(conf *coreconfig.Configuration) { + if conf.IOWeightCgroupLevelConfigFile == "" { + general.Errorf("IOWeightCgroupLevelConfigFile isn't configured") + return + } + + ioWightCgroupLevelConfigs := make(map[string]uint64) + err := general.LoadJsonConfig(conf.IOWeightCgroupLevelConfigFile, &ioWightCgroupLevelConfigs) + if err != nil { + general.Errorf("load IOWeightCgroupLevelConfig failed with error: %v", err) + return + } + + for relativeCgPath, weight := range ioWightCgroupLevelConfigs { + err := cgroupmgr.ApplyIOWeightWithRelativePath(relativeCgPath, defaultDevID, weight) + if err != nil { + general.Errorf("ApplyIOWeightWithRelativePath for devID: %s in relativeCgPath: %s failed with error: %v", + defaultDevID, relativeCgPath, err) + } else { + general.Infof("ApplyIOWeightWithRelativePath for devID: %s, weight: %d in relativeCgPath: %s successfully", + defaultDevID, weight, relativeCgPath) + } + } +} + func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) { if conf.IOWeightQoSLevelConfigFile == "" { @@ -68,7 +98,6 @@ func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, general.Warningf("no QoSLevelToDefaultValue in extraControlKnobConfigs") continue } - for _, containerStatus := range pod.Status.ContainerStatuses { podUID, containerID := string(pod.UID), native.TrimContainerIDPrefix(containerStatus.ContainerID) err := cgroupmgr.ApplyUnifiedDataForContainer(podUID, containerID, extraControlKnobConfigs[controlKnobKeyIOWeight].CgroupSubsysName, cgroupIOWeightName, qosLevelDefaultValue) @@ -111,4 +140,9 @@ func IOWeightTaskFunc(conf *coreconfig.Configuration, if len(conf.IOWeightQoSLevelConfigFile) > 0 { applyIOWeightQoSLevelConfig(conf, emitter, metaServer) } + + // checking cgroup-level io.weight configuration. + if len(conf.IOWeightCgroupLevelConfigFile) > 0 { + applyIOWeightCgroupLevelConfig(conf) + } } diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go new file mode 100644 index 0000000000..d125ae0cc2 --- /dev/null +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux_test.go @@ -0,0 +1,166 @@ +//go:build linux +// +build linux + +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ioweight + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + coreconfig "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/config/agent" + configagent "github.com/kubewharf/katalyst-core/pkg/config/agent" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" + "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + metaagent "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" + "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric" + "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" + "github.com/kubewharf/katalyst-core/pkg/metrics" + "github.com/kubewharf/katalyst-core/pkg/util/machine" +) + +func makeMetaServer() (*metaserver.MetaServer, error) { + server := &metaserver.MetaServer{ + MetaAgent: &metaagent.MetaAgent{}, + } + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 1, 2) + if err != nil { + return nil, err + } + + server.KatalystMachineInfo = &machine.KatalystMachineInfo{ + CPUTopology: cpuTopology, + } + server.MetricsFetcher = metric.NewFakeMetricsFetcher(metrics.DummyMetrics{}) + return server, nil +} + +func TestIOWeightTaskFunc(t *testing.T) { + t.Parallel() + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: false, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, nil, nil) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, nil) + + metaServer, err := makeMetaServer() + assert.NoError(t, err) + metaServer.PodFetcher = &pod.PodFetcherStub{PodList: []*v1.Pod{}} + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, nil, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + normalPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "normalPod", + Name: "normalPod", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c", + }, + }, + }, + } + + metaServer.PodFetcher = &pod.PodFetcherStub{PodList: []*v1.Pod{normalPod}} + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) + + IOWeightTaskFunc(&coreconfig.Configuration{ + AgentConfiguration: &agent.AgentConfiguration{ + StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ + QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{ + IOQRMPluginConfig: &qrm.IOQRMPluginConfig{ + IOWeightOption: qrm.IOWeightOption{ + EnableSettingIOWeight: true, + IOWeightQoSLevelConfigFile: "fake", + IOWeightCgroupLevelConfigFile: "fake", + }, + }, + }, + }, + }, + }, metrics.DummyMetrics{}, &dynamicconfig.DynamicAgentConfiguration{}, metrics.DummyMetrics{}, metaServer) +} diff --git a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go index 4d3aace918..f637e6e010 100644 --- a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go +++ b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go @@ -122,7 +122,7 @@ func (p *StaticPolicy) Start() (err error) { } if p.enableSettingWBT { general.Infof("setWBT enabled") - err := periodicalhandler.RegisterPeriodicalHandler(qrm.QRMMemoryPluginPeriodicalHandlerGroupName, + err := periodicalhandler.RegisterPeriodicalHandler(qrm.QRMIOPluginPeriodicalHandlerGroupName, dirtymem.EnableSetDirtyMemPeriodicalHandlerName, dirtymem.SetDirtyMem, 300*time.Second) if err != nil { general.Infof("setSockMem failed, err=%v", err) diff --git a/pkg/config/agent/qrm/io_plugin.go b/pkg/config/agent/qrm/io_plugin.go index 1184431e70..c74329ff02 100644 --- a/pkg/config/agent/qrm/io_plugin.go +++ b/pkg/config/agent/qrm/io_plugin.go @@ -31,8 +31,9 @@ type WritebackThrottlingOption struct { } type IOWeightOption struct { - EnableSettingIOWeight bool - IOWeightQoSLevelConfigFile string + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string + IOWeightCgroupLevelConfigFile string } func NewIOQRMPluginConfig() *IOQRMPluginConfig {