From 4be86c6894a34dcbb1ff4fea915c57f2cc7545e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=81=A5=E4=BF=9E?= Date: Mon, 4 Mar 2024 17:50:50 +0800 Subject: [PATCH] feat(qrm): memory plugin support shared_cores with numa_binding --- pkg/agent/qrm-plugins/commonstate/types.go | 2 +- .../policy_allocation_handlers.go | 26 ++++++++++++------- .../dynamicpolicy/policy_hint_handlers.go | 18 ++++++++----- .../memory/dynamicpolicy/state/state.go | 7 +++-- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/pkg/agent/qrm-plugins/commonstate/types.go b/pkg/agent/qrm-plugins/commonstate/types.go index 22bc59820..3b7b46964 100644 --- a/pkg/agent/qrm-plugins/commonstate/types.go +++ b/pkg/agent/qrm-plugins/commonstate/types.go @@ -23,7 +23,7 @@ import "github.com/kubewharf/katalyst-core/pkg/util/general" // 2. applied by QRM framework according to ociPropertyName // // there may be new types of control knobs, -// we won't modified this struct to identify them, +// we won't modify this struct to identify them, // and we will register custom per-control-knob executor to deal with them. type ControlKnobInfo struct { ControlKnobValue string `json:"control_knob_value"` diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go index 1a9f76c42..8c211e280 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go @@ -40,7 +40,13 @@ func (p *DynamicPolicy) sharedCoresAllocationHandler(ctx context.Context, return nil, fmt.Errorf("sharedCoresAllocationHandler got nil request") } - return p.allocateNUMAsWithoutNUMABindingPods(ctx, req, apiconsts.PodAnnotationQoSLevelSharedCores) + switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { + case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: + return p.numaBindingAllocationHandler(ctx, req, apiconsts.PodAnnotationQoSLevelSharedCores) + default: + return p.allocateNUMAsWithoutNUMABindingPods(ctx, req, apiconsts.PodAnnotationQoSLevelSharedCores) + } + } func (p *DynamicPolicy) reclaimedCoresAllocationHandler(ctx context.Context, @@ -64,16 +70,16 @@ func (p *DynamicPolicy) dedicatedCoresAllocationHandler(ctx context.Context, switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: - return p.dedicatedCoresWithNUMABindingAllocationHandler(ctx, req) + return p.numaBindingAllocationHandler(ctx, req, apiconsts.PodAnnotationQoSLevelDedicatedCores) default: return p.dedicatedCoresWithoutNUMABindingAllocationHandler(ctx, req) } } -func (p *DynamicPolicy) dedicatedCoresWithNUMABindingAllocationHandler(ctx context.Context, - req *pluginapi.ResourceRequest) (*pluginapi.ResourceAllocationResponse, error) { +func (p *DynamicPolicy) numaBindingAllocationHandler(ctx context.Context, + req *pluginapi.ResourceRequest, qosLevel string) (*pluginapi.ResourceAllocationResponse, error) { if req.ContainerType == pluginapi.ContainerType_SIDECAR { - return p.dedicatedCoresWithNUMABindingAllocationSidecarHandler(ctx, req) + return p.numaBindingAllocationSidecarHandler(ctx, req, qosLevel) } reqInt, _, err := util.GetQuantityFromResourceReq(req) @@ -170,7 +176,7 @@ func (p *DynamicPolicy) dedicatedCoresWithNUMABindingAllocationHandler(ctx conte TopologyAwareAllocations: topologyAwareAllocations, Labels: general.DeepCopyMap(req.Labels), Annotations: general.DeepCopyMap(req.Annotations), - QoSLevel: apiconsts.PodAnnotationQoSLevelDedicatedCores, + QoSLevel: qosLevel, } p.state.SetAllocationInfo(v1.ResourceMemory, req.PodUid, req.ContainerName, allocationInfo) @@ -203,10 +209,10 @@ func (p *DynamicPolicy) dedicatedCoresWithoutNUMABindingAllocationHandler(_ cont return nil, fmt.Errorf("not support dedicated_cores without NUMA binding") } -// dedicatedCoresWithNUMABindingAllocationSidecarHandler allocates for sidecar +// numaBindingAllocationSidecarHandler allocates for sidecar // currently, we set cpuset of sidecar to the cpuset of its main container -func (p *DynamicPolicy) dedicatedCoresWithNUMABindingAllocationSidecarHandler(_ context.Context, - req *pluginapi.ResourceRequest) (*pluginapi.ResourceAllocationResponse, error) { +func (p *DynamicPolicy) numaBindingAllocationSidecarHandler(_ context.Context, + req *pluginapi.ResourceRequest, qosLevel string) (*pluginapi.ResourceAllocationResponse, error) { podResourceEntries := p.state.GetPodResourceEntries() podEntries := podResourceEntries[v1.ResourceMemory] @@ -238,7 +244,7 @@ func (p *DynamicPolicy) dedicatedCoresWithNUMABindingAllocationSidecarHandler(_ TopologyAwareAllocations: nil, // not count sidecar quantity Labels: general.DeepCopyMap(req.Labels), Annotations: general.DeepCopyMap(req.Annotations), - QoSLevel: apiconsts.PodAnnotationQoSLevelDedicatedCores, + QoSLevel: qosLevel, } // update pod entries directly. if one of subsequent steps is failed, diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go index 48c0b4c65..a08278db4 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go @@ -33,16 +33,20 @@ import ( qosutil "github.com/kubewharf/katalyst-core/pkg/util/qos" ) -func (p *DynamicPolicy) sharedCoresHintHandler(_ context.Context, +func (p *DynamicPolicy) sharedCoresHintHandler(ctx context.Context, req *pluginapi.ResourceRequest) (*pluginapi.ResourceHintsResponse, error) { if req == nil { return nil, fmt.Errorf("got nil request") } - return util.PackResourceHintsResponse(req, string(v1.ResourceMemory), - map[string]*pluginapi.ListOfTopologyHints{ - string(v1.ResourceMemory): nil, // indicates that there is no numa preference - }) + if !qosutil.AnnotationsIndicateNUMABinding(req.Annotations) { + return util.PackResourceHintsResponse(req, string(v1.ResourceMemory), + map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceMemory): nil, // indicates that there is no numa preference + }) + } + + return p.numaBindingHintHandler(ctx, req) } func (p *DynamicPolicy) reclaimedCoresHintHandler(ctx context.Context, @@ -58,13 +62,13 @@ func (p *DynamicPolicy) dedicatedCoresHintHandler(ctx context.Context, switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: - return p.dedicatedCoresWithNUMABindingHintHandler(ctx, req) + return p.numaBindingHintHandler(ctx, req) default: return p.dedicatedCoresWithoutNUMABindingHintHandler(ctx, req) } } -func (p *DynamicPolicy) dedicatedCoresWithNUMABindingHintHandler(_ context.Context, +func (p *DynamicPolicy) numaBindingHintHandler(_ context.Context, req *pluginapi.ResourceRequest) (*pluginapi.ResourceHintsResponse, error) { // currently, we set cpuset of sidecar to the cpuset of its main container, // so there is no numa preference here. diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/state/state.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/state/state.go index 8a9c76074..ba60ae204 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/state/state.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/state/state.go @@ -126,11 +126,10 @@ func (ai *AllocationInfo) Clone() *AllocationInfo { return clone } -// CheckNumaBinding returns true if the AllocationInfo is for pod with -// dedicated-qos and numa-binding enhancement +// CheckNumaBinding returns true if the AllocationInfo is for pod with numa-binding enhancement func (ai *AllocationInfo) CheckNumaBinding() bool { - return ai.QoSLevel == consts.PodAnnotationQoSLevelDedicatedCores && - ai.Annotations[consts.PodAnnotationMemoryEnhancementNumaBinding] == consts.PodAnnotationMemoryEnhancementNumaBindingEnable + return ai.Annotations[consts.PodAnnotationMemoryEnhancementNumaBinding] == + consts.PodAnnotationMemoryEnhancementNumaBindingEnable } // CheckMainContainer returns true if the AllocationInfo is for main container