From 6ea10526aab917d61096607c9b985fa50dbcb8d7 Mon Sep 17 00:00:00 2001 From: Robin Lu Date: Wed, 28 Feb 2024 10:18:53 +0800 Subject: [PATCH] feat: adding support to qos level io.weight setting Signed-off-by: Robin Lu --- .../app/options/qrm/io_plugin.go | 16 +++ .../qrm-plugins/io/handlers/ioweight/const.go | 26 ++++ .../io/handlers/ioweight/ioweight_linux.go | 114 ++++++++++++++++++ .../qrm-plugins/io/staticpolicy/policy.go | 34 ++++-- pkg/config/agent/qrm/io_plugin.go | 6 + 5 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 pkg/agent/qrm-plugins/io/handlers/ioweight/const.go create mode 100644 pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go diff --git a/cmd/katalyst-agent/app/options/qrm/io_plugin.go b/cmd/katalyst-agent/app/options/qrm/io_plugin.go index 520c6abd0..7e6bd6773 100644 --- a/cmd/katalyst-agent/app/options/qrm/io_plugin.go +++ b/cmd/katalyst-agent/app/options/qrm/io_plugin.go @@ -28,6 +28,7 @@ type IOOptions struct { WritebackThrottlingOption // option for writeback throttling, it determin the recycling speed of dirty memory. // TO-DO //DirtyThrottlingOption // option for dirty throttling, it determin the global watermark of dirty memory. + IOWeightOption } type WritebackThrottlingOption struct { @@ -36,6 +37,11 @@ type WritebackThrottlingOption struct { WBTValueSSD int } +type IOWeightOption struct { + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string +} + func NewIOOptions() *IOOptions { return &IOOptions{ PolicyName: "static", @@ -44,6 +50,10 @@ func NewIOOptions() *IOOptions { WBTValueHDD: 75000, WBTValueSSD: 2000, }, + IOWeightOption: IOWeightOption{ + EnableSettingIOWeight: false, + IOWeightQoSLevelConfigFile: "", + }, } } @@ -58,6 +68,10 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.WBTValueHDD, "writeback throttling value for HDD") fs.IntVar(&o.WBTValueSSD, "disk-wbt-ssd", o.WBTValueSSD, "writeback throttling value for SSD") + fs.BoolVar(&o.EnableSettingIOWeight, "enable-io-weight", + o.EnableSettingIOWeight, "if set it to true, io.weight related control operations will be executed") + fs.StringVar(&o.IOWeightQoSLevelConfigFile, "io-weight-qos-config-file", + o.IOWeightQoSLevelConfigFile, "the absolute path of io.weight qos config file") } func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { @@ -65,5 +79,7 @@ func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error { conf.EnableSettingWBT = o.EnableSettingWBT conf.WBTValueHDD = o.WBTValueHDD conf.WBTValueSSD = o.WBTValueSSD + conf.EnableSettingIOWeight = o.EnableSettingIOWeight + conf.IOWeightQoSLevelConfigFile = o.IOWeightQoSLevelConfigFile return nil } diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go new file mode 100644 index 000000000..a73498ba3 --- /dev/null +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/const.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ioweight + +const EnableSetIOWeightPeriodicalHandlerName = "SetIOWeight" + +const ( + metricNameIOWeight = "async_handler_io_weight" + + controlKnobKeyIOWeight = "io_weight" + cgroupIOWeightName = "io.weight" +) diff --git a/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go new file mode 100644 index 000000000..faccd408f --- /dev/null +++ b/pkg/agent/qrm-plugins/io/handlers/ioweight/ioweight_linux.go @@ -0,0 +1,114 @@ +//go:build linux +// +build linux + +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ioweight + +import ( + "context" + + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/commonstate" + coreconfig "github.com/kubewharf/katalyst-core/pkg/config" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + "github.com/kubewharf/katalyst-core/pkg/metrics" + "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common" + cgroupmgr "github.com/kubewharf/katalyst-core/pkg/util/cgroup/manager" + "github.com/kubewharf/katalyst-core/pkg/util/general" + "github.com/kubewharf/katalyst-core/pkg/util/native" +) + +func applyIOWeightQoSLevelConfig(conf *coreconfig.Configuration, + emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) { + if conf.IOWeightQoSLevelConfigFile == "" { + general.Infof("no IOWeightQoSLevelConfigFile found") + return + } + + var extraControlKnobConfigs commonstate.ExtraControlKnobConfigs + if err := general.LoadJsonConfig(conf.IOWeightQoSLevelConfigFile, &extraControlKnobConfigs); err != nil { + general.Errorf("IOWeightQoSLevelConfigFile load failed:%v", err) + return + } + ctx := context.Background() + podList, err := metaServer.GetPodList(ctx, native.PodIsActive) + if err != nil { + general.Infof("get pod list failed: %v", err) + return + } + + for _, pod := range podList { + if pod == nil { + general.Warningf("get nil pod from metaServer") + continue + } + qosConfig := conf.QoSConfiguration + qosLevel, err := qosConfig.GetQoSLevelForPod(pod) + if err != nil { + general.Warningf("GetQoSLevelForPod failed:%v", err) + continue + } + qosLevelDefaultValue, ok := extraControlKnobConfigs[controlKnobKeyIOWeight].QoSLevelToDefaultValue[qosLevel] + if !ok { + general.Warningf("no QoSLevelToDefaultValue in extraControlKnobConfigs") + continue + } + + for _, containerStatus := range pod.Status.ContainerStatuses { + podUID, containerID := string(pod.UID), native.TrimContainerIDPrefix(containerStatus.ContainerID) + err := cgroupmgr.ApplyUnifiedDataForContainer(podUID, containerID, extraControlKnobConfigs[controlKnobKeyIOWeight].CgroupSubsysName, cgroupIOWeightName, qosLevelDefaultValue) + if err != nil { + general.Warningf("ApplyUnifiedDataForContainer failed:%v", err) + continue + } + } + } +} + +func IOWeightTaskFunc(conf *coreconfig.Configuration, + _ interface{}, _ *dynamicconfig.DynamicAgentConfiguration, + emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) { + general.Infof("called") + + if conf == nil { + general.Errorf("nil extraConf") + return + } else if emitter == nil { + general.Errorf("nil emitter") + return + } else if metaServer == nil { + general.Errorf("nil metaServer") + return + } + + // SettingIOWeight featuregate. + if !conf.EnableSettingIOWeight { + general.Infof("EnableSettingIOWeight disabled") + return + } + + if !common.CheckCgroup2UnifiedMode() { + general.Infof("skip IOWeightTaskFunc in cg1 env") + return + } + + // checking qos-level io.weight configuration. + if len(conf.IOWeightQoSLevelConfigFile) > 0 { + applyIOWeightQoSLevelConfig(conf, emitter, metaServer) + } +} diff --git a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go index 7a4c39fe4..4d3aace91 100644 --- a/pkg/agent/qrm-plugins/io/staticpolicy/policy.go +++ b/pkg/agent/qrm-plugins/io/staticpolicy/policy.go @@ -29,9 +29,11 @@ import ( "github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/agent" "github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/agent/qrm" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/io/handlers/dirtymem" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/io/handlers/ioweight" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" "github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler" "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/config/generic" "github.com/kubewharf/katalyst-core/pkg/metaserver" "github.com/kubewharf/katalyst-core/pkg/metrics" "github.com/kubewharf/katalyst-core/pkg/util/general" @@ -46,14 +48,17 @@ const ( type StaticPolicy struct { sync.Mutex - name string - stopCh chan struct{} - started bool + name string + stopCh chan struct{} + started bool + qosConfig *generic.QoSConfiguration + emitter metrics.MetricEmitter metaServer *metaserver.MetaServer agentCtx *agent.GenericContext - enableSettingWBT bool + enableSettingWBT bool + enableSettingIOWeight bool } // NewStaticPolicy returns a static io policy @@ -65,12 +70,14 @@ func NewStaticPolicy(agentCtx *agent.GenericContext, conf *config.Configuration, }) policyImplement := &StaticPolicy{ - emitter: wrappedEmitter, - metaServer: agentCtx.MetaServer, - agentCtx: agentCtx, - stopCh: make(chan struct{}), - name: fmt.Sprintf("%s_%s", agentName, IOResourcePluginPolicyNameStatic), - enableSettingWBT: conf.EnableSettingWBT, + emitter: wrappedEmitter, + metaServer: agentCtx.MetaServer, + agentCtx: agentCtx, + stopCh: make(chan struct{}), + name: fmt.Sprintf("%s_%s", agentName, IOResourcePluginPolicyNameStatic), + qosConfig: conf.QoSConfiguration, + enableSettingWBT: conf.EnableSettingWBT, + enableSettingIOWeight: conf.EnableSettingIOWeight, } // todo: currently there is no resource needed to be topology-aware and synchronously allocated in this plugin, @@ -106,6 +113,13 @@ func (p *StaticPolicy) Start() (err error) { _ = p.emitter.StoreInt64(util.MetricNameHeartBeat, 1, metrics.MetricTypeNameRaw) }, time.Second*30, p.stopCh) + if p.enableSettingIOWeight { + err = periodicalhandler.RegisterPeriodicalHandler(qrm.QRMIOPluginPeriodicalHandlerGroupName, + ioweight.EnableSetIOWeightPeriodicalHandlerName, ioweight.IOWeightTaskFunc, 30*time.Second) + if err != nil { + general.Infof("register syncIOWeight failed, err=%v", err) + } + } if p.enableSettingWBT { general.Infof("setWBT enabled") err := periodicalhandler.RegisterPeriodicalHandler(qrm.QRMMemoryPluginPeriodicalHandlerGroupName, diff --git a/pkg/config/agent/qrm/io_plugin.go b/pkg/config/agent/qrm/io_plugin.go index 335141925..1184431e7 100644 --- a/pkg/config/agent/qrm/io_plugin.go +++ b/pkg/config/agent/qrm/io_plugin.go @@ -21,6 +21,7 @@ type IOQRMPluginConfig struct { PolicyName string WritebackThrottlingOption + IOWeightOption } type WritebackThrottlingOption struct { @@ -29,6 +30,11 @@ type WritebackThrottlingOption struct { WBTValueSSD int } +type IOWeightOption struct { + EnableSettingIOWeight bool + IOWeightQoSLevelConfigFile string +} + func NewIOQRMPluginConfig() *IOQRMPluginConfig { return &IOQRMPluginConfig{} }