Skip to content

Commit

Permalink
Clean up resourceSlices during exit
Browse files Browse the repository at this point in the history
Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>
  • Loading branch information
ArangoGutierrez committed Oct 23, 2024
1 parent 322a3a7 commit c50aa21
Showing 1 changed file with 35 additions and 1 deletion.
36 changes: 35 additions & 1 deletion cmd/nvidia-dra-controller/imex.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"

v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -94,6 +97,10 @@ func manageResourceSlices(ctx context.Context, clientset kubernetes.Interface, o
return fmt.Errorf("error starting resource slice controller: %w", err)
}

// Setup signal catching
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)

go func() {
for {
select {
Expand All @@ -109,7 +116,12 @@ func manageResourceSlices(ctx context.Context, clientset kubernetes.Interface, o
delete(newDriverResources.Pools, removedDomain)
controller.Update(&newDriverResources)
driverResources = newDriverResources
case <-ctx.Done():
case <-sigs:
controller.Stop()
err = cleanupImexChannels(ctx, clientset)
if err != nil {
klog.Errorf("error cleaning up resource slices: %v", err)
}
return
}
}
Expand Down Expand Up @@ -259,3 +271,25 @@ func generateImexChannelPool(imexDomain string, numChannels int) resourceslice.P

return pool
}

// cleanupImexChannels removes all resource slices created by the IMEX manager.
func cleanupImexChannels(ctx context.Context, clientset kubernetes.Interface) error {
// Delete all resource slices created by the IMEX manager
ops := metav1.ListOptions{
FieldSelector: fmt.Sprintf("%s=%s", resourceapi.ResourceSliceSelectorDriver, DriverName),
}
l, err := clientset.ResourceV1alpha3().ResourceSlices().List(ctx, ops)
if err != nil {
return fmt.Errorf("error listing resource slices: %w", err)
}

for _, rs := range l.Items {
klog.Info("Deleting resource slice: ", rs.Name)
err := clientset.ResourceV1alpha3().ResourceSlices().Delete(ctx, rs.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("error deleting resource slice %s: %w", rs.Name, err)
}
}

return nil
}

0 comments on commit c50aa21

Please sign in to comment.