diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go index e263316f4..04f4823eb 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go @@ -249,6 +249,8 @@ func TestAllocate(t *testing.T) { description string req *pluginapi.ResourceRequest expectedResp *pluginapi.ResourceAllocationResponse + enableReclaim bool + wantError bool cpuTopology *machine.CPUTopology enhancementDefaultValues map[string]string }{ @@ -668,6 +670,158 @@ func TestAllocate(t *testing.T) { }, cpuTopology: cpuTopology, }, + { + description: "req for shared_cores numa_binding main container with enableReclaim false", + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: &pluginapi.TopologyHint{ + Nodes: []uint64{0}, + Preferred: true, + }, + }, + expectedResp: &pluginapi.ResourceAllocationResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + AllocationResult: &pluginapi.ResourceAllocation{ + ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ + string(v1.ResourceCPU): { + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 3, + AllocationResult: machine.NewCPUSet(1, 8, 9).String(), + ResourceHints: &pluginapi.ListOfTopologyHints{ + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{0}, + Preferred: true, + }, + }, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + cpuTopology: cpuTopology, + }, + { + description: "req for shared_cores numa_binding main container with enableReclaim true", + enableReclaim: true, + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 0.3, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: &pluginapi.TopologyHint{ + Nodes: []uint64{0}, + Preferred: true, + }, + }, + expectedResp: &pluginapi.ResourceAllocationResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + AllocationResult: &pluginapi.ResourceAllocation{ + ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ + string(v1.ResourceCPU): { + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 1, + AllocationResult: machine.NewCPUSet(1).String(), + ResourceHints: &pluginapi.ListOfTopologyHints{ + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{0}, + Preferred: true, + }, + }, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + cpuTopology: cpuTopology, + }, + { + description: "req for shared_cores numa_binding main container with invalid hint", + wantError: true, + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 0.3, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: &pluginapi.TopologyHint{ + Nodes: []uint64{0, 1}, + Preferred: true, + }, + }, + expectedResp: nil, + cpuTopology: cpuTopology, + }, } for _, tc := range testCases { @@ -681,9 +835,17 @@ func TestAllocate(t *testing.T) { dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues } + if tc.enableReclaim { + dynamicPolicy.dynamicConfig.GetDynamicConfiguration().EnableReclaim = true + } + resp, err := dynamicPolicy.Allocate(context.Background(), tc.req) - as.Nil(err) + if tc.wantError { + as.NotNil(err) + continue + } + as.Nil(err) tc.expectedResp.PodUid = tc.req.PodUid as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) @@ -703,6 +865,8 @@ func TestGetTopologyHints(t *testing.T) { testCases := []struct { description string req *pluginapi.ResourceRequest + podEntries state.PodEntries + cpuNUMAHintPreferPolicy string expectedResp *pluginapi.ResourceHintsResponse cpuTopology *machine.CPUTopology enhancementDefaultValues map[string]string @@ -1084,17 +1248,558 @@ func TestGetTopologyHints(t *testing.T) { }, cpuTopology: cpuTopology, }, - } - - for _, tc := range testCases { - tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyHints") - as.Nil(err) - - dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) - as.Nil(err) - - if tc.enhancementDefaultValues != nil { - dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues + { + description: "req for shared_cores with numa_binding main container with default cpuNUMAHintPreferPolicy(spreading)", + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 1, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + }, + podEntries: state.PodEntries{ + "373d08e4-7a6b-4293-aaaf-b135ff812kkk": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812kkk", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameDedicated, + AllocationResult: machine.MustParse("1,8,9"), + OriginalAllocationResult: machine.MustParse("1,8,9"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 8, 9), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 8, 9), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + "373d08e4-7a6b-4293-aaaf-b135ff8123bf": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "ec6e2f30-c78a-4bc4-9576-c916db5281a3": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "2432d068-c5a0-46ba-a7bd-b69d9bd16961": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "2432d068-c5a0-46ba-a7bd-b69d9bd16961", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("9,11,13,15"), + OriginalAllocationResult: machine.MustParse("9,11,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + state.PoolNameReclaim: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameReclaim, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("9,11,13,15"), + OriginalAllocationResult: machine.MustParse("9,11,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + }, + }, + state.PoolNameShare: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameShare, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + }, + }, + "share-NUMA1": state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameShare, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("3,10"), + OriginalAllocationResult: machine.MustParse("3,10"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + "373d08e4-7a6b-4293-aaaf-b135ff812aaa": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA1", + AllocationResult: machine.MustParse("3,10"), + OriginalAllocationResult: machine.MustParse("3,10"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1, + }, + }, + }, + expectedResp: &pluginapi.ResourceHintsResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceCPU): { + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{1}, + Preferred: false, + }, + { + Nodes: []uint64{2}, + Preferred: true, + }, + { + Nodes: []uint64{3}, + Preferred: true, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + cpuTopology: cpuTopology, + }, + { + description: "req for shared_cores with numa_binding main container with packing cpuNUMAHintPreferPolicy(packing)", + cpuNUMAHintPreferPolicy: cpuconsts.CPUNUMAHintPreferPolicyPacking, + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 1, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + }, + podEntries: state.PodEntries{ + "373d08e4-7a6b-4293-aaaf-b135ff812kkk": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812kkk", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameDedicated, + AllocationResult: machine.MustParse("1,8,9"), + OriginalAllocationResult: machine.MustParse("1,8,9"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 8, 9), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 8, 9), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + "373d08e4-7a6b-4293-aaaf-b135ff8123bf": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "ec6e2f30-c78a-4bc4-9576-c916db5281a3": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "2432d068-c5a0-46ba-a7bd-b69d9bd16961": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "2432d068-c5a0-46ba-a7bd-b69d9bd16961", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("9,11,13,15"), + OriginalAllocationResult: machine.MustParse("9,11,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + state.PoolNameReclaim: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameReclaim, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("9,11,13,15"), + OriginalAllocationResult: machine.MustParse("9,11,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(9), + 1: machine.NewCPUSet(11), + 2: machine.NewCPUSet(13), + 3: machine.NewCPUSet(15), + }, + }, + }, + state.PoolNameShare: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameShare, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4-5,12,6-7,14"), + OriginalAllocationResult: machine.MustParse("4-5,12,6-7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5, 12), + 3: machine.NewCPUSet(6, 7, 14), + }, + }, + }, + "share-NUMA1": state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameShare, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("3,10"), + OriginalAllocationResult: machine.MustParse("3,10"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + "373d08e4-7a6b-4293-aaaf-b135ff812aaa": state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA1", + AllocationResult: machine.MustParse("3,10"), + OriginalAllocationResult: machine.MustParse("3,10"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 10), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1, + }, + }, + }, + expectedResp: &pluginapi.ResourceHintsResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceCPU): { + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{1}, + Preferred: true, + }, + { + Nodes: []uint64{2}, + Preferred: false, + }, + { + Nodes: []uint64{3}, + Preferred: false, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + cpuTopology: cpuTopology, + }, + } + + for _, tc := range testCases { + tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyHints") + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) + as.Nil(err) + + if tc.enhancementDefaultValues != nil { + dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues + } + + if tc.podEntries != nil { + machineState, err := generateMachineStateFromPodEntries(tc.cpuTopology, tc.podEntries) + as.Nil(err) + + dynamicPolicy.state.SetPodEntries(tc.podEntries) + dynamicPolicy.state.SetMachineState(machineState) + } + + if tc.cpuNUMAHintPreferPolicy != "" { + dynamicPolicy.cpuNUMAHintPreferPolicy = tc.cpuNUMAHintPreferPolicy } resp, err := dynamicPolicy.GetTopologyHints(context.Background(), tc.req) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go index b60595e43..4cae128fe 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go @@ -20,9 +20,11 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "strings" "testing" + "k8s.io/apimachinery/pkg/util/sets" pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing" @@ -31,6 +33,8 @@ import ( "github.com/stretchr/testify/require" "github.com/kubewharf/katalyst-api/pkg/consts" + cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor" "github.com/kubewharf/katalyst-core/pkg/util/machine" ) @@ -2539,3 +2543,561 @@ func TestGetSocketTopology(t *testing.T) { as.Equalf(tc.expectedSocketTopology, actualSocketToplogy, "failed in test case: %s", tc.description) } } +func TestAllocationInfo_GetSpecifiedNUMABindingPoolName(t *testing.T) { + testName := "test" + + type fields struct { + PodUid string + PodNamespace string + PodName string + ContainerName string + ContainerType string + ContainerIndex uint64 + RampUp bool + OwnerPoolName string + PodRole string + PodType string + AllocationResult machine.CPUSet + OriginalAllocationResult machine.CPUSet + TopologyAwareAssignments map[int]machine.CPUSet + OriginalTopologyAwareAssignments map[int]machine.CPUSet + InitTimestamp string + Labels map[string]string + Annotations map[string]string + QoSLevel string + RequestQuantity float64 + } + tests := []struct { + name string + fields fields + want string + wantErr bool + }{ + { + name: "shared_cores with numa_binding pod get pool name normally", + fields: fields{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + cpuconsts.CPUStateAnnotationKeyNUMAHint: "0", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1, + }, + want: "share-NUMA0", + wantErr: false, + }, + { + name: "dedicated_cores with numa_binding pod get pool name failed", + fields: fields{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 1, + }, + want: "", + wantErr: true, + }, + { + name: "shared_cores with numa_binding pod without hint annotation get pool name failed", + fields: fields{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1, + }, + want: "", + wantErr: true, + }, + { + name: "shared_cores with numa_binding pod with invalid hint annotation get pool name failed", + fields: fields{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + cpuconsts.CPUStateAnnotationKeyNUMAHint: "0-1", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1, + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ai := &AllocationInfo{ + PodUid: tt.fields.PodUid, + PodNamespace: tt.fields.PodNamespace, + PodName: tt.fields.PodName, + ContainerName: tt.fields.ContainerName, + ContainerType: tt.fields.ContainerType, + ContainerIndex: tt.fields.ContainerIndex, + RampUp: tt.fields.RampUp, + OwnerPoolName: tt.fields.OwnerPoolName, + PodRole: tt.fields.PodRole, + PodType: tt.fields.PodType, + AllocationResult: tt.fields.AllocationResult, + OriginalAllocationResult: tt.fields.OriginalAllocationResult, + TopologyAwareAssignments: tt.fields.TopologyAwareAssignments, + OriginalTopologyAwareAssignments: tt.fields.OriginalTopologyAwareAssignments, + InitTimestamp: tt.fields.InitTimestamp, + Labels: tt.fields.Labels, + Annotations: tt.fields.Annotations, + QoSLevel: tt.fields.QoSLevel, + RequestQuantity: tt.fields.RequestQuantity, + } + got, err := ai.GetSpecifiedNUMABindingPoolName() + if (err != nil) != tt.wantErr { + t.Errorf("AllocationInfo.GetSpecifiedNUMABindingPoolName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("AllocationInfo.GetSpecifiedNUMABindingPoolName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCheckNUMABindingSharedCoresAntiAffinity(t *testing.T) { + testName := "test" + type args struct { + ai *AllocationInfo + annotations map[string]string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "anti affinity with dedicated numa_binding pods", + args: args{ + ai: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + want: true, + }, + { + name: "anti affinity with shared_cores numa_binding pods with same specified pool name", + args: args{ + ai: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationCPUEnhancementCPUSet: "batch", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationCPUEnhancementCPUSet: "bmq", + }, + }, + want: true, + }, + { + name: "not anti affinity with shared_cores numa_binding pods with same specified pool name", + args: args{ + ai: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1"), + OriginalAllocationResult: machine.MustParse("1"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationCPUEnhancementCPUSet: "batch", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationCPUEnhancementCPUSet: "batch", + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckNUMABindingSharedCoresAntiAffinity(tt.args.ai, tt.args.annotations); got != tt.want { + t.Errorf("CheckNUMABindingSharedCoresAntiAffinity() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPodEntries_GetFilteredPoolsCPUSetMap(t *testing.T) { + testName := "test" + type args struct { + ignorePools sets.String + } + tests := []struct { + name string + pe PodEntries + args args + want map[string]map[int]machine.CPUSet + wantErr bool + }{ + { + name: "get filtered pools cpuset map normally", + pe: PodEntries{ + "373d08e4-7a6b-4293-aaaf-b135ff8123bf": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "ec6e2f30-c78a-4bc4-9576-c916db5281a3": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "2432d068-c5a0-46ba-a7bd-b69d9bd16961": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "2432d068-c5a0-46ba-a7bd-b69d9bd16961", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameReclaim, + AllocationResult: machine.MustParse("5,8,10,13,15"), + OriginalAllocationResult: machine.MustParse("5,8,10,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8), + 1: machine.NewCPUSet(10), + 2: machine.NewCPUSet(5, 13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8), + 1: machine.NewCPUSet(10), + 2: machine.NewCPUSet(5, 13), + 3: machine.NewCPUSet(15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + PoolNameReclaim: ContainerEntries{ + "": &AllocationInfo{ + PodUid: PoolNameReclaim, + OwnerPoolName: PoolNameReclaim, + AllocationResult: machine.MustParse("5,8,10,13,15"), + OriginalAllocationResult: machine.MustParse("5,8,10,13,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8), + 1: machine.NewCPUSet(10), + 2: machine.NewCPUSet(5, 13), + 3: machine.NewCPUSet(15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8), + 1: machine.NewCPUSet(10), + 2: machine.NewCPUSet(5, 13), + 3: machine.NewCPUSet(15), + }, + }, + }, + PoolNameShare: ContainerEntries{ + "": &AllocationInfo{ + PodUid: PoolNameShare, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + }, + }, + "share-NUMA3": ContainerEntries{ + "": &AllocationInfo{ + PodUid: PoolNameShare, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("6,7,14"), + OriginalAllocationResult: machine.MustParse("6,7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + "373d08e4-7a6b-4293-aaaf-b135ff812aaa": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA3", + AllocationResult: machine.MustParse("6,7,14"), + OriginalAllocationResult: machine.MustParse("6,7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + }, + args: args{ + ignorePools: ResidentPools, + }, + want: map[string]map[int]machine.CPUSet{ + "share": { + cpuadvisor.FakedNUMAID: machine.MustParse("1,3-4,9,11-12"), + }, + "share-NUMA3": { + 3: machine.MustParse("6,7,14"), + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.pe.GetFilteredPoolsCPUSetMap(tt.args.ignorePools) + if (err != nil) != tt.wantErr { + t.Errorf("PodEntries.GetFilteredPoolsCPUSetMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("PodEntries.GetFilteredPoolsCPUSetMap() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util_test.go index c2f7bda3c..13d0f1256 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util_test.go @@ -22,6 +22,7 @@ package state import ( "io/ioutil" "os" + "reflect" "testing" "github.com/stretchr/testify/require" @@ -29,6 +30,7 @@ import ( "github.com/kubewharf/katalyst-api/pkg/consts" cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor" "github.com/kubewharf/katalyst-core/pkg/util/machine" ) @@ -562,3 +564,298 @@ func TestGetSpecifiedPoolName(t *testing.T) { }) } } + +func TestCountAllocationInfosToPoolsQuantityMap(t *testing.T) { + testName := "test" + SetContainerRequestedCores(func(allocationInfo *AllocationInfo) float64 { + return allocationInfo.RequestQuantity + }) + + type args struct { + allocationInfos []*AllocationInfo + poolsQuantityMap map[string]map[int]int + } + tests := []struct { + name string + args args + want map[string]map[int]int + wantErr bool + }{ + // TODO: Add test cases. + { + name: "count allocationInfos to pools quantity map normally", + args: args{ + allocationInfos: []*AllocationInfo{ + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA3", + AllocationResult: machine.MustParse("6,7,14"), + OriginalAllocationResult: machine.MustParse("6,7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812bbb", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + cpuconsts.CPUStateAnnotationKeyNUMAHint: "3", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 3.1, + }, + }, + poolsQuantityMap: map[string]map[int]int{}, + }, + want: map[string]map[int]int{ + "share-NUMA3": { + 3: 5, + }, + "share": { + cpuadvisor.FakedNUMAID: 4, + }, + }, + + wantErr: false, + }, + { + name: "count allocationInfos to pools quantity map with invalid hint", + args: args{ + allocationInfos: []*AllocationInfo{ + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA3", + AllocationResult: machine.MustParse("6,7,14"), + OriginalAllocationResult: machine.MustParse("6,7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812bbb", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + cpuconsts.CPUStateAnnotationKeyNUMAHint: "2-3", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 3.1, + }, + }, + poolsQuantityMap: map[string]map[int]int{}, + }, + wantErr: true, + }, + { + name: "count allocationInfos to pools quantity map with nil poolsQuantityMap", + args: args{ + allocationInfos: []*AllocationInfo{ + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812aaa", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: "share-NUMA3", + AllocationResult: machine.MustParse("6,7,14"), + OriginalAllocationResult: machine.MustParse("6,7,14"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7, 14), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff812bbb", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + cpuconsts.CPUStateAnnotationKeyNUMAHint: "2-3", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 1.1, + }, + { + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("1,3-4,9,11-12"), + OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1, 9), + 1: machine.NewCPUSet(3, 11), + 2: machine.NewCPUSet(4, 12), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 3.1, + }, + }, + poolsQuantityMap: nil, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CountAllocationInfosToPoolsQuantityMap(tt.args.allocationInfos, tt.args.poolsQuantityMap); (err != nil) != tt.wantErr { + t.Errorf("CountAllocationInfosToPoolsQuantityMap() error = %v, wantErr %v", err, tt.wantErr) + } else if err == nil { + if !reflect.DeepEqual(tt.args.poolsQuantityMap, tt.want) { + t.Errorf("CountAllocationInfosToPoolsQuantityMap() = %v, want %v", tt.args.poolsQuantityMap, tt.want) + } + } + }) + } +} diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go index 3ffca1743..87f85aa53 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go @@ -817,6 +817,66 @@ func TestAllocate(t *testing.T) { }, }, }, + { + description: "req for shared_cores with numa_binding main container", + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + Hint: &pluginapi.TopologyHint{ + Nodes: []uint64{0}, + Preferred: true, + }, + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + expectedResp: &pluginapi.ResourceAllocationResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + AllocationResult: &pluginapi.ResourceAllocation{ + ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ + string(v1.ResourceMemory): { + OciPropertyName: util.OCIPropertyNameCPUSetMems, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 2147483648, + AllocationResult: machine.NewCPUSet(0).String(), + ResourceHints: &pluginapi.ListOfTopologyHints{ + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{0}, + Preferred: true, + }, + }, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + }, + }, } for _, tc := range testCases { @@ -1232,6 +1292,65 @@ func TestGetTopologyHints(t *testing.T) { }, }, }, + { + description: "req for shared_cores with numa_binding main container", + req: &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 1073741824, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + expectedResp: &pluginapi.ResourceHintsResponse{ + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceMemory): { + Hints: []*pluginapi.TopologyHint{ + { + Nodes: []uint64{0}, + Preferred: true, + }, + { + Nodes: []uint64{1}, + Preferred: true, + }, + { + Nodes: []uint64{2}, + Preferred: true, + }, + { + Nodes: []uint64{3}, + Preferred: true, + }, + }, + }, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, + }, + }, } for _, tc := range testCases {