feat: support upgrade image builder pod

bentoml · Jan 12, 2023 · bccd922 · bccd922
1 parent 337f26f
commit bccd922
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 8 deletions.
diff --git a/controllers/resources/bentorequest_controller.go b/controllers/resources/bentorequest_controller.go
@@ -18,6 +18,8 @@ package resources
 
 import (
 	"context"
+	"strconv"
+
 	// nolint: gosec
 	"crypto/md5"
 	"encoding/hex"
@@ -26,6 +28,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
@@ -72,6 +75,10 @@ import (
 	"github.com/aws/aws-sdk-go/service/ecr"
 )
 
+const (
+	KubeAnnotationBentoRequestHash = "yatai.ai/bento-request-hash"
+)
+
 // BentoRequestReconciler reconciles a BentoRequest object
 type BentoRequestReconciler struct {
 	client.Client
@@ -264,6 +271,28 @@ func (r *BentoRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			return
 		}
 
+		if !podIsNotFound {
+			oldHash := pod.Annotations[KubeAnnotationBentoRequestHash]
+			var hash uint64
+			hash, err = hashstructure.Hash(bentoRequest.Spec, hashstructure.FormatV2, nil)
+			if err != nil {
+				err = errors.Wrap(err, "get bentoRequest CR spec hash")
+				return
+			}
+			hashStr := strconv.FormatUint(hash, 10)
+			if oldHash != hashStr {
+				err = r.Delete(ctx, pod)
+				if err != nil {
+					err = errors.Wrapf(err, "delete pod %s", podName)
+					return
+				}
+				result = ctrl.Result{
+					Requeue: true,
+				}
+				return
+			}
+		}
+
 		if podIsNotFound {
 			var imageInfo ImageInfo
 			imageInfo, err = r.getImageInfo(ctx, GetImageInfoOption{
@@ -276,10 +305,9 @@ func (r *BentoRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request
 
 			r.Recorder.Eventf(bentoRequest, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Generating image builder pod: %s", podName)
 			pod, err = r.generateImageBuilderPod(ctx, GenerateImageBuilderPodOption{
-				PodName:          podName,
-				ImageInfo:        imageInfo,
-				BentoRequest:     bentoRequest,
-				RecreateIfFailed: true,
+				PodName:      podName,
+				ImageInfo:    imageInfo,
+				BentoRequest: bentoRequest,
 			})
 			if err != nil {
 				err = errors.Wrap(err, "create image builder pod")
@@ -921,10 +949,9 @@ func (r *BentoRequestReconciler) getBento(ctx context.Context, bentoRequest *res
 }
 
 type GenerateImageBuilderPodOption struct {
-	ImageInfo        ImageInfo
-	PodName          string
-	BentoRequest     *resourcesv1alpha1.BentoRequest
-	RecreateIfFailed bool
+	ImageInfo    ImageInfo
+	PodName      string
+	BentoRequest *resourcesv1alpha1.BentoRequest
 }
 
 func (r *BentoRequestReconciler) generateImageBuilderPod(ctx context.Context, opt GenerateImageBuilderPodOption) (pod *corev1.Pod, err error) {
@@ -1443,6 +1470,14 @@ echo "Done"
 	}
 
 	kubeAnnotations := make(map[string]string)
+	var hash uint64
+	hash, err = hashstructure.Hash(opt.BentoRequest.Spec, hashstructure.FormatV2, nil)
+	if err != nil {
+		err = errors.Wrap(err, "get bentoRequest CR spec hash")
+		return
+	}
+	hashStr := strconv.FormatUint(hash, 10)
+	kubeAnnotations[KubeAnnotationBentoRequestHash] = hashStr
 	var command []string
 	args := []string{
 		"--context=/workspace/buildcontext",

diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451
 	github.com/huandu/xstrings v1.3.2
 	github.com/iancoleman/strcase v0.2.0
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.19.0

diff --git a/go.sum b/go.sum
@@ -297,6 +297,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=