Skip to content

Commit 838ba52

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #6378 from BigDarkClown/multitaint
Taint utils taking multiple taints
2 parents 7ea57f3 + c4063ef commit 838ba52

File tree

2 files changed

+132
-70
lines changed

2 files changed

+132
-70
lines changed

‎cluster-autoscaler/utils/taints/taints.go

Lines changed: 51 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -151,16 +151,12 @@ func (tc TaintConfig) isExplicitlyReportedTaint(taint string) bool {
151151
return ok
152152
}
153153

154-
// getKeyShortName converts taint key to short name for logging
155-
func getKeyShortName(key string) string {
156-
switch key {
157-
case ToBeDeletedTaint:
158-
return "ToBeDeletedTaint"
159-
case DeletionCandidateTaint:
160-
return "DeletionCandidateTaint"
161-
default:
162-
return key
154+
func taintKeys(taints []apiv1.Taint) []string {
155+
var keys []string
156+
for _, taint := range taints {
157+
keys = append(keys, taint.Key)
163158
}
159+
return keys
164160
}
165161

166162
// MarkToBeDeleted sets a taint that makes the node unschedulable.
@@ -170,7 +166,7 @@ func MarkToBeDeleted(node *apiv1.Node, client kube_client.Interface, cordonNode
170166
Value: fmt.Sprint(time.Now().Unix()),
171167
Effect: apiv1.TaintEffectNoSchedule,
172168
}
173-
return AddTaint(node, client, taint, cordonNode)
169+
return AddTaints(node, client, []apiv1.Taint{taint}, cordonNode)
174170
}
175171

176172
// MarkDeletionCandidate sets a soft taint that makes the node preferably unschedulable.
@@ -180,11 +176,11 @@ func MarkDeletionCandidate(node *apiv1.Node, client kube_client.Interface) error
180176
Value: fmt.Sprint(time.Now().Unix()),
181177
Effect: apiv1.TaintEffectPreferNoSchedule,
182178
}
183-
return AddTaint(node, client, taint, false)
179+
return AddTaints(node, client, []apiv1.Taint{taint}, false)
184180
}
185181

186-
// AddTaint sets the specified taint on the node.
187-
func AddTaint(node *apiv1.Node, client kube_client.Interface, taint apiv1.Taint, cordonNode bool) error {
182+
// AddTaints sets the specified taints on the node.
183+
func AddTaints(node *apiv1.Node, client kube_client.Interface, taints []apiv1.Taint, cordonNode bool) error {
188184
retryDeadline := time.Now().Add(maxRetryDeadline)
189185
freshNode := node.DeepCopy()
190186
var err error
@@ -194,12 +190,12 @@ func AddTaint(node *apiv1.Node, client kube_client.Interface, taint apiv1.Taint,
194190
// Get the newest version of the node.
195191
freshNode, err = client.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
196192
if err != nil || freshNode == nil {
197-
klog.Warningf("Error while adding %v taint on node %v: %v", getKeyShortName(taint.Key), node.Name, err)
193+
klog.Warningf("Error while adding %v taints on node %v: %v", strings.Join(taintKeys(taints), ","), node.Name, err)
198194
return fmt.Errorf("failed to get node %v: %v", node.Name, err)
199195
}
200196
}
201197

202-
if !addTaintToSpec(freshNode, taint, cordonNode) {
198+
if !addTaintsToSpec(freshNode, taints, cordonNode) {
203199
if !refresh {
204200
// Make sure we have the latest version before skipping update.
205201
refresh = true
@@ -215,22 +211,27 @@ func AddTaint(node *apiv1.Node, client kube_client.Interface, taint apiv1.Taint,
215211
}
216212

217213
if err != nil {
218-
klog.Warningf("Error while adding %v taint on node %v: %v", getKeyShortName(taint.Key), node.Name, err)
214+
klog.Warningf("Error while adding %v taints on node %v: %v", strings.Join(taintKeys(taints), ","), node.Name, err)
219215
return err
220216
}
221-
klog.V(1).Infof("Successfully added %v on node %v", getKeyShortName(taint.Key), node.Name)
217+
klog.V(1).Infof("Successfully added %v on node %v", strings.Join(taintKeys(taints), ","), node.Name)
222218
return nil
223219
}
224220
}
225221

226-
func addTaintToSpec(node *apiv1.Node, taint apiv1.Taint, cordonNode bool) bool {
227-
for _, t := range node.Spec.Taints {
228-
if t.Key == taint.Key {
229-
klog.V(2).Infof("%v already present on node %v, t: %v", taint.Key, node.Name, t)
230-
return false
222+
func addTaintsToSpec(node *apiv1.Node, taints []apiv1.Taint, cordonNode bool) bool {
223+
taintsAdded := false
224+
for _, taint := range taints {
225+
if HasTaint(node, taint.Key) {
226+
klog.V(2).Infof("%v already present on node %v", taint.Key, node.Name)
227+
continue
231228
}
229+
taintsAdded = true
230+
node.Spec.Taints = append(node.Spec.Taints, taint)
231+
}
232+
if !taintsAdded {
233+
return false
232234
}
233-
node.Spec.Taints = append(node.Spec.Taints, taint)
234235
if cordonNode {
235236
klog.V(1).Infof("Marking node %v to be cordoned by Cluster Autoscaler", node.Name)
236237
node.Spec.Unschedulable = true
@@ -285,16 +286,16 @@ func GetTaintTime(node *apiv1.Node, taintKey string) (*time.Time, error) {
285286

286287
// CleanToBeDeleted cleans CA's NoSchedule taint from a node.
287288
func CleanToBeDeleted(node *apiv1.Node, client kube_client.Interface, cordonNode bool) (bool, error) {
288-
return CleanTaint(node, client, ToBeDeletedTaint, cordonNode)
289+
return CleanTaints(node, client, []string{ToBeDeletedTaint}, cordonNode)
289290
}
290291

291292
// CleanDeletionCandidate cleans CA's soft NoSchedule taint from a node.
292293
func CleanDeletionCandidate(node *apiv1.Node, client kube_client.Interface) (bool, error) {
293-
return CleanTaint(node, client, DeletionCandidateTaint, false)
294+
return CleanTaints(node, client, []string{DeletionCandidateTaint}, false)
294295
}
295296

296-
// CleanTaint cleans the specified taint from a node.
297-
func CleanTaint(node *apiv1.Node, client kube_client.Interface, taintKey string, cordonNode bool) (bool, error) {
297+
// CleanTaints cleans the specified taints from a node.
298+
func CleanTaints(node *apiv1.Node, client kube_client.Interface, taintKeys []string, cordonNode bool) (bool, error) {
298299
retryDeadline := time.Now().Add(maxRetryDeadline)
299300
freshNode := node.DeepCopy()
300301
var err error
@@ -304,15 +305,21 @@ func CleanTaint(node *apiv1.Node, client kube_client.Interface, taintKey string,
304305
// Get the newest version of the node.
305306
freshNode, err = client.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
306307
if err != nil || freshNode == nil {
307-
klog.Warningf("Error while adding %v taint on node %v: %v", getKeyShortName(taintKey), node.Name, err)
308+
klog.Warningf("Error while removing %v taints from node %v: %v", strings.Join(taintKeys, ","), node.Name, err)
308309
return false, fmt.Errorf("failed to get node %v: %v", node.Name, err)
309310
}
310311
}
311312
newTaints := make([]apiv1.Taint, 0)
312313
for _, taint := range freshNode.Spec.Taints {
313-
if taint.Key == taintKey {
314-
klog.V(1).Infof("Releasing taint %+v on node %v", taint, node.Name)
315-
} else {
314+
keepTaint := true
315+
for _, taintKey := range taintKeys {
316+
if taint.Key == taintKey {
317+
klog.V(1).Infof("Releasing taint %+v on node %v", taint, node.Name)
318+
keepTaint = false
319+
break
320+
}
321+
}
322+
if keepTaint {
316323
newTaints = append(newTaints, taint)
317324
}
318325
}
@@ -339,37 +346,41 @@ func CleanTaint(node *apiv1.Node, client kube_client.Interface, taintKey string,
339346
}
340347

341348
if err != nil {
342-
klog.Warningf("Error while releasing %v taint on node %v: %v", getKeyShortName(taintKey), node.Name, err)
349+
klog.Warningf("Error while releasing %v taints on node %v: %v", strings.Join(taintKeys, ","), node.Name, err)
343350
return false, err
344351
}
345-
klog.V(1).Infof("Successfully released %v on node %v", getKeyShortName(taintKey), node.Name)
352+
klog.V(1).Infof("Successfully released %v on node %v", strings.Join(taintKeys, ","), node.Name)
346353
return true, nil
347354
}
348355
}
349356

350357
// CleanAllToBeDeleted cleans ToBeDeleted taints from given nodes.
351358
func CleanAllToBeDeleted(nodes []*apiv1.Node, client kube_client.Interface, recorder kube_record.EventRecorder, cordonNode bool) {
352-
CleanAllTaints(nodes, client, recorder, ToBeDeletedTaint, cordonNode)
359+
CleanAllTaints(nodes, client, recorder, []string{ToBeDeletedTaint}, cordonNode)
353360
}
354361

355362
// CleanAllDeletionCandidates cleans DeletionCandidate taints from given nodes.
356363
func CleanAllDeletionCandidates(nodes []*apiv1.Node, client kube_client.Interface, recorder kube_record.EventRecorder) {
357-
CleanAllTaints(nodes, client, recorder, DeletionCandidateTaint, false)
364+
CleanAllTaints(nodes, client, recorder, []string{DeletionCandidateTaint}, false)
358365
}
359366

360367
// CleanAllTaints cleans all specified taints from given nodes.
361-
func CleanAllTaints(nodes []*apiv1.Node, client kube_client.Interface, recorder kube_record.EventRecorder, taintKey string, cordonNode bool) {
368+
func CleanAllTaints(nodes []*apiv1.Node, client kube_client.Interface, recorder kube_record.EventRecorder, taintKeys []string, cordonNode bool) {
362369
for _, node := range nodes {
363-
if !HasTaint(node, taintKey) {
370+
taintsPresent := false
371+
for _, taintKey := range taintKeys {
372+
taintsPresent = taintsPresent || HasTaint(node, taintKey)
373+
}
374+
if !taintsPresent {
364375
continue
365376
}
366-
cleaned, err := CleanTaint(node, client, taintKey, cordonNode)
377+
cleaned, err := CleanTaints(node, client, taintKeys, cordonNode)
367378
if err != nil {
368379
recorder.Eventf(node, apiv1.EventTypeWarning, "ClusterAutoscalerCleanup",
369-
"failed to clean %v on node %v: %v", getKeyShortName(taintKey), node.Name, err)
380+
"failed to clean %v on node %v: %v", strings.Join(taintKeys, ","), node.Name, err)
370381
} else if cleaned {
371382
recorder.Eventf(node, apiv1.EventTypeNormal, "ClusterAutoscalerCleanup",
372-
"removed %v taint from node %v", getKeyShortName(taintKey), node.Name)
383+
"removed %v taints from node %v", strings.Join(taintKeys, ","), node.Name)
373384
}
374385
}
375386
}

‎cluster-autoscaler/utils/taints/taints_test.go

Lines changed: 81 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -67,28 +67,43 @@ func TestSoftMarkNodes(t *testing.T) {
6767
func TestCheckNodes(t *testing.T) {
6868
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
6969
node := BuildTestNode("node", 1000, 1000)
70-
taint := apiv1.Taint{
71-
Key: ToBeDeletedTaint,
72-
Value: fmt.Sprint(time.Now().Unix()),
73-
Effect: apiv1.TaintEffectNoSchedule,
70+
taints := []apiv1.Taint{
71+
{
72+
Key: ToBeDeletedTaint,
73+
Value: fmt.Sprint(time.Now().Unix()),
74+
Effect: apiv1.TaintEffectNoSchedule,
75+
},
76+
{
77+
Key: "other-taint",
78+
Value: fmt.Sprint(time.Now().Unix()),
79+
Effect: apiv1.TaintEffectNoSchedule,
80+
},
7481
}
75-
addTaintToSpec(node, taint, false)
82+
addTaintsToSpec(node, taints, false)
7683
fakeClient := buildFakeClientWithConflicts(t, node)
7784

7885
updatedNode := getNode(t, fakeClient, "node")
7986
assert.True(t, HasToBeDeletedTaint(updatedNode))
87+
assert.True(t, HasTaint(node, "other-taint"))
8088
assert.False(t, HasDeletionCandidateTaint(updatedNode))
8189
}
8290

8391
func TestSoftCheckNodes(t *testing.T) {
8492
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
8593
node := BuildTestNode("node", 1000, 1000)
86-
taint := apiv1.Taint{
87-
Key: DeletionCandidateTaint,
88-
Value: fmt.Sprint(time.Now().Unix()),
89-
Effect: apiv1.TaintEffectPreferNoSchedule,
94+
taints := []apiv1.Taint{
95+
{
96+
Key: DeletionCandidateTaint,
97+
Value: fmt.Sprint(time.Now().Unix()),
98+
Effect: apiv1.TaintEffectPreferNoSchedule,
99+
},
100+
{
101+
Key: "other-taint",
102+
Value: fmt.Sprint(time.Now().Unix()),
103+
Effect: apiv1.TaintEffectPreferNoSchedule,
104+
},
90105
}
91-
addTaintToSpec(node, taint, false)
106+
addTaintsToSpec(node, taints, false)
92107
fakeClient := buildFakeClientWithConflicts(t, node)
93108

94109
updatedNode := getNode(t, fakeClient, "node")
@@ -131,16 +146,24 @@ func TestSoftQueryNodes(t *testing.T) {
131146
func TestCleanNodes(t *testing.T) {
132147
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
133148
node := BuildTestNode("node", 1000, 1000)
134-
taint := apiv1.Taint{
135-
Key: ToBeDeletedTaint,
136-
Value: fmt.Sprint(time.Now().Unix()),
137-
Effect: apiv1.TaintEffectNoSchedule,
149+
taints := []apiv1.Taint{
150+
{
151+
Key: ToBeDeletedTaint,
152+
Value: fmt.Sprint(time.Now().Unix()),
153+
Effect: apiv1.TaintEffectNoSchedule,
154+
},
155+
{
156+
Key: "other-taint",
157+
Value: fmt.Sprint(time.Now().Unix()),
158+
Effect: apiv1.TaintEffectNoSchedule,
159+
},
138160
}
139-
addTaintToSpec(node, taint, false)
161+
addTaintsToSpec(node, taints, false)
140162
fakeClient := buildFakeClientWithConflicts(t, node)
141163

142164
updatedNode := getNode(t, fakeClient, "node")
143165
assert.True(t, HasToBeDeletedTaint(updatedNode))
166+
assert.True(t, HasTaint(updatedNode, "other-taint"))
144167
assert.False(t, updatedNode.Spec.Unschedulable)
145168

146169
cleaned, err := CleanToBeDeleted(node, fakeClient, false)
@@ -150,22 +173,31 @@ func TestCleanNodes(t *testing.T) {
150173
updatedNode = getNode(t, fakeClient, "node")
151174
assert.NoError(t, err)
152175
assert.False(t, HasToBeDeletedTaint(updatedNode))
176+
assert.True(t, HasTaint(updatedNode, "other-taint"))
153177
assert.False(t, updatedNode.Spec.Unschedulable)
154178
}
155179

156180
func TestCleanNodesWithCordon(t *testing.T) {
157181
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
158182
node := BuildTestNode("node", 1000, 1000)
159-
taint := apiv1.Taint{
160-
Key: ToBeDeletedTaint,
161-
Value: fmt.Sprint(time.Now().Unix()),
162-
Effect: apiv1.TaintEffectNoSchedule,
183+
taints := []apiv1.Taint{
184+
{
185+
Key: ToBeDeletedTaint,
186+
Value: fmt.Sprint(time.Now().Unix()),
187+
Effect: apiv1.TaintEffectNoSchedule,
188+
},
189+
{
190+
Key: "other-taint",
191+
Value: fmt.Sprint(time.Now().Unix()),
192+
Effect: apiv1.TaintEffectNoSchedule,
193+
},
163194
}
164-
addTaintToSpec(node, taint, true)
195+
addTaintsToSpec(node, taints, true)
165196
fakeClient := buildFakeClientWithConflicts(t, node)
166197

167198
updatedNode := getNode(t, fakeClient, "node")
168199
assert.True(t, HasToBeDeletedTaint(updatedNode))
200+
assert.True(t, HasTaint(updatedNode, "other-taint"))
169201
assert.True(t, updatedNode.Spec.Unschedulable)
170202

171203
cleaned, err := CleanToBeDeleted(node, fakeClient, true)
@@ -175,22 +207,31 @@ func TestCleanNodesWithCordon(t *testing.T) {
175207
updatedNode = getNode(t, fakeClient, "node")
176208
assert.NoError(t, err)
177209
assert.False(t, HasToBeDeletedTaint(updatedNode))
210+
assert.True(t, HasTaint(updatedNode, "other-taint"))
178211
assert.False(t, updatedNode.Spec.Unschedulable)
179212
}
180213

181214
func TestCleanNodesWithCordonOnOff(t *testing.T) {
182215
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
183216
node := BuildTestNode("node", 1000, 1000)
184-
taint := apiv1.Taint{
185-
Key: ToBeDeletedTaint,
186-
Value: fmt.Sprint(time.Now().Unix()),
187-
Effect: apiv1.TaintEffectNoSchedule,
217+
taints := []apiv1.Taint{
218+
{
219+
Key: ToBeDeletedTaint,
220+
Value: fmt.Sprint(time.Now().Unix()),
221+
Effect: apiv1.TaintEffectPreferNoSchedule,
222+
},
223+
{
224+
Key: "other-taint",
225+
Value: fmt.Sprint(time.Now().Unix()),
226+
Effect: apiv1.TaintEffectPreferNoSchedule,
227+
},
188228
}
189-
addTaintToSpec(node, taint, true)
229+
addTaintsToSpec(node, taints, true)
190230
fakeClient := buildFakeClientWithConflicts(t, node)
191231

192232
updatedNode := getNode(t, fakeClient, "node")
193233
assert.True(t, HasToBeDeletedTaint(updatedNode))
234+
assert.True(t, HasTaint(updatedNode, "other-taint"))
194235
assert.True(t, updatedNode.Spec.Unschedulable)
195236

196237
cleaned, err := CleanToBeDeleted(node, fakeClient, false)
@@ -200,22 +241,31 @@ func TestCleanNodesWithCordonOnOff(t *testing.T) {
200241
updatedNode = getNode(t, fakeClient, "node")
201242
assert.NoError(t, err)
202243
assert.False(t, HasToBeDeletedTaint(updatedNode))
244+
assert.True(t, HasTaint(updatedNode, "other-taint"))
203245
assert.True(t, updatedNode.Spec.Unschedulable)
204246
}
205247

206248
func TestSoftCleanNodes(t *testing.T) {
207249
defer setConflictRetryInterval(setConflictRetryInterval(time.Millisecond))
208250
node := BuildTestNode("node", 1000, 1000)
209-
taint := apiv1.Taint{
210-
Key: DeletionCandidateTaint,
211-
Value: fmt.Sprint(time.Now().Unix()),
212-
Effect: apiv1.TaintEffectPreferNoSchedule,
251+
taints := []apiv1.Taint{
252+
{
253+
Key: DeletionCandidateTaint,
254+
Value: fmt.Sprint(time.Now().Unix()),
255+
Effect: apiv1.TaintEffectPreferNoSchedule,
256+
},
257+
{
258+
Key: "other-taint",
259+
Value: fmt.Sprint(time.Now().Unix()),
260+
Effect: apiv1.TaintEffectPreferNoSchedule,
261+
},
213262
}
214-
addTaintToSpec(node, taint, false)
263+
addTaintsToSpec(node, taints, false)
215264
fakeClient := buildFakeClientWithConflicts(t, node)
216265

217266
updatedNode := getNode(t, fakeClient, "node")
218267
assert.True(t, HasDeletionCandidateTaint(updatedNode))
268+
assert.True(t, HasTaint(updatedNode, "other-taint"))
219269

220270
cleaned, err := CleanDeletionCandidate(node, fakeClient)
221271
assert.True(t, cleaned)
@@ -224,6 +274,7 @@ func TestSoftCleanNodes(t *testing.T) {
224274
updatedNode = getNode(t, fakeClient, "node")
225275
assert.NoError(t, err)
226276
assert.False(t, HasDeletionCandidateTaint(updatedNode))
277+
assert.True(t, HasTaint(updatedNode, "other-taint"))
227278
}
228279

229280
func TestCleanAllToBeDeleted(t *testing.T) {

0 commit comments

Comments
 (0)