Merge pull request #144 from syntasso/update-delete-flat-files

Update delete flat files

api/v1alpha1/destination_types.go

@@ -62,16 +62,16 @@ type DestinationSpec struct {
 }
 
 const (
-	//if modifying these dont forget to edit below where they are written as a
-	//kubebuilder comment for setting the default and Enum values.
-	FilepathExpressionTypeNone             = "none"
-	FilepathExpressionTypeNestedByMetadata = "nestedByMetadata"
+	// if modifying these dont forget to edit below where they are written as a
+	// kubebuilder comment for setting the default and Enum values.
+	FilepathModeNone             = "none"
+	FilepathModeNestedByMetadata = "nestedByMetadata"
 )
 
 type Filepath struct {
 	// +kubebuilder:validation:Enum:={nestedByMetadata,none}
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="filepath.mode is immutable"
-	// The  of filepathExpression, either:
+	// filepath.mode can be set to either:
 	// - nestedByMetadata (default): files from the pipeline will be placed in a nested directory structure
 	// - none: file from the pipeline will be placed in a flat directory structure
 	// filepath.mode is immutable
@@ -80,9 +80,9 @@ type Filepath struct {
 
 // it gets defaulted by the K8s API, but for unit testing it wont be defaulted
 // since its not a real k8s api, so it may be empty when running unit tests.
-func (d *Destination) GetFilepathExpressionType() string {
+func (d *Destination) GetFilepathMode() string {
 	if d.Spec.Filepath.Mode == "" {
-		return FilepathExpressionTypeNestedByMetadata
+		return FilepathModeNestedByMetadata
 	}
 	return d.Spec.Filepath.Mode
 }

config/crd/bases/platform.kratix.io_destinations.yaml

@@ -48,7 +48,7 @@ spec:
                 properties:
                   mode:
                     description: |-
-                      The  of filepathExpression, either:
+                      filepath.mode can be set to either:
                       - nestedByMetadata (default): files from the pipeline will be placed in a nested directory structure
                       - none: file from the pipeline will be placed in a flat directory structure
                       filepath.mode is immutable

controllers/destination_controller.go

@@ -18,21 +18,23 @@ package controllers
 
 import (
 	"context"
-	"path/filepath"
-
-	"k8s.io/apimachinery/pkg/api/errors"
-
+	"fmt"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	"path/filepath"
 	"sigs.k8s.io/yaml"
 
+	"k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/go-logr/logr"
 	"github.com/syntasso/kratix/api/v1alpha1"
 	"github.com/syntasso/kratix/lib/writers"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const canaryWorkload = "kratix-canary"
+
 // DestinationReconciler reconciles a Destination object
 type DestinationReconciler struct {
 	Client    client.Client
@@ -117,10 +119,9 @@ func (r *DestinationReconciler) createResourcePathWithExample(writer writers.Sta
 	}
 	nsBytes, _ := yaml.Marshal(kratixConfigMap)
 
-	return writer.WriteDirWithObjects(writers.PreserveExistingContentsInDir, resourcesDir, v1alpha1.Workload{
-		Filepath: "kratix-canary-configmap.yaml",
-		Content:  string(nsBytes),
-	})
+	return writer.UpdateFiles(canaryWorkload, []v1alpha1.Workload{{
+		Filepath: fmt.Sprintf("%s/kratix-canary-configmap.yaml", resourcesDir),
+		Content:  string(nsBytes)}}, nil)
 }
 
 func (r *DestinationReconciler) createDependenciesPathWithExample(writer writers.StateStoreWriter) error {
@@ -133,10 +134,9 @@ func (r *DestinationReconciler) createDependenciesPathWithExample(writer writers
 	}
 	nsBytes, _ := yaml.Marshal(kratixNamespace)
 
-	return writer.WriteDirWithObjects(writers.PreserveExistingContentsInDir, dependenciesDir, v1alpha1.Workload{
-		Filepath: "kratix-canary-namespace.yaml",
-		Content:  string(nsBytes),
-	})
+	return writer.UpdateFiles(canaryWorkload, []v1alpha1.Workload{{
+		Filepath: fmt.Sprintf("%s/kratix-canary-namespace.yaml", dependenciesDir),
+		Content:  string(nsBytes)}}, nil)
 }
 
 // SetupWithManager sets up the controller with the Manager.

controllers/workplacement_controller.go

@@ -18,11 +18,13 @@ package controllers
 
 import (
 	"context"
-	"path/filepath"
-
+	"errors"
+	"fmt"
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/api/errors"
+	"gopkg.in/yaml.v2"
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+	"path/filepath"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -37,6 +39,10 @@ const (
 	dependenciesDir = "dependencies"
 )
 
+type StateFile struct {
+	Files []string `json:"files"`
+}
+
 // WorkPlacementReconciler reconciles a WorkPlacement object
 type WorkPlacementReconciler struct {
 	Client client.Client
@@ -61,7 +67,7 @@ func (r *WorkPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	workPlacement := &v1alpha1.WorkPlacement{}
 	err := r.Client.Get(context.Background(), req.NamespacedName, workPlacement)
 	if err != nil {
-		if errors.IsNotFound(err) {
+		if k8sErrors.IsNotFound(err) {
 			return ctrl.Result{}, nil
 		}
 		logger.Error(err, "Error getting WorkPlacement", "workPlacement", req.Name)
@@ -87,14 +93,14 @@ func (r *WorkPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	//Mock this out
 	writer, err := newWriter(opts, *destination)
 	if err != nil {
-		if errors.IsNotFound(err) {
+		if k8sErrors.IsNotFound(err) {
 			return defaultRequeue, nil
 		}
 		return ctrl.Result{}, err
 	}
 
 	if !workPlacement.DeletionTimestamp.IsZero() {
-		return r.deleteWorkPlacement(ctx, writer, workPlacement, logger)
+		return r.deleteWorkPlacement(ctx, writer, workPlacement, destination.GetFilepathMode(), logger)
 	}
 
 	if resourceutil.FinalizersAreMissing(workPlacement, workPlacementFinalizers) {
@@ -110,13 +116,30 @@ func (r *WorkPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	return ctrl.Result{}, nil
 }
 
-func (r *WorkPlacementReconciler) deleteWorkPlacement(ctx context.Context, writer writers.StateStoreWriter, workPlacement *v1alpha1.WorkPlacement, logger logr.Logger) (ctrl.Result, error) {
+func (r *WorkPlacementReconciler) deleteWorkPlacement(ctx context.Context, writer writers.StateStoreWriter, workPlacement *v1alpha1.WorkPlacement, filePathMode string, logger logr.Logger) (ctrl.Result, error) {
 	if !controllerutil.ContainsFinalizer(workPlacement, repoCleanupWorkPlacementFinalizer) {
 		return ctrl.Result{}, nil
 	}
+	logger.Info("cleaning up work on repository", "workplacement", workPlacement.Name)
+
+	var err error
+	if filePathMode == v1alpha1.FilepathModeNone {
+		var kratixFile []byte
+		kratixFilePath := fmt.Sprintf(".kratix/%s-%s.yaml", workPlacement.Namespace, workPlacement.Name)
+		if kratixFile, err = writer.ReadFile(kratixFilePath); err != nil {
+			logger.Error(err, "failed to read .kratix state file")
+			return defaultRequeue, err
+		}
+		stateFile := StateFile{}
+		if err = yaml.Unmarshal(kratixFile, &stateFile); err != nil {
+			logger.Error(err, "failed to unmarshal .kratix state file")
+			return defaultRequeue, err
+		}
+		err = writer.UpdateFiles(workPlacement.Name, nil, append(stateFile.Files, kratixFilePath))
+	} else {
+		err = writer.UpdateInDir(getDir(*workPlacement)+"/", workPlacement.Name, nil)
+	}
 
-	logger.Info("cleaning up files on repository", "repository", workPlacement.Name)
-	err := r.removeWorkFromRepository(writer, *workPlacement, logger)
 	if err != nil {
 		logger.Error(err, "error removing work from repository, will try again in 5 seconds")
 		return defaultRequeue, err
@@ -131,27 +154,68 @@ func (r *WorkPlacementReconciler) deleteWorkPlacement(ctx context.Context, write
 }
 
 func (r *WorkPlacementReconciler) writeWorkloadsToStateStore(writer writers.StateStoreWriter, workPlacement v1alpha1.WorkPlacement, destination v1alpha1.Destination, logger logr.Logger) error {
-	dir := getDir(workPlacement)
-	if destination.GetFilepathExpressionType() == v1alpha1.FilepathExpressionTypeNone {
-		dir = ""
+	var err error
+	if destination.GetFilepathMode() == v1alpha1.FilepathModeNone {
+		var kratixFile []byte
+		if kratixFile, err = writer.ReadFile(fmt.Sprintf(".kratix/%s-%s.yaml", workPlacement.Namespace, workPlacement.Name)); ignoreNotFound(err) != nil {
+			return fmt.Errorf("failed to read .kratix state file: %s", err)
+		}
+		oldStateFile := StateFile{}
+		if err = yaml.Unmarshal(kratixFile, &oldStateFile); err != nil {
+			return fmt.Errorf("failed to unmarshal .kratix state file: %s", err)
+		}
+
+		newStateFile := StateFile{
+			Files: workloadsFilenames(workPlacement.Spec.Workloads),
+		}
+		stateFileContent, marshalErr := yaml.Marshal(newStateFile)
+		if marshalErr != nil {
+			return fmt.Errorf("failed to marshal new .kratix state file: %s", err)
+		}
+
+		stateFileWorkload := v1alpha1.Workload{
+			Filepath: fmt.Sprintf(".kratix/%s-%s.yaml", workPlacement.Namespace, workPlacement.Name),
+			Content:  string(stateFileContent),
+		}
+		err = writer.UpdateFiles(workPlacement.Name, append(workPlacement.Spec.Workloads, stateFileWorkload), cleanupWorkloads(oldStateFile.Files, workPlacement.Spec.Workloads))
+	} else {
+		err = writer.UpdateInDir(getDir(workPlacement), workPlacement.Name, workPlacement.Spec.Workloads)
 	}
-	err := writer.WriteDirWithObjects(writers.DeleteExistingContentsInDir, dir, workPlacement.Spec.Workloads...)
+
 	if err != nil {
 		logger.Error(err, "Error writing resources to repository")
 		return err
 	}
-
 	return nil
 }
 
-func (r *WorkPlacementReconciler) removeWorkFromRepository(writer writers.StateStoreWriter, workPlacement v1alpha1.WorkPlacement, logger logr.Logger) error {
-	//MinIO needs a trailing slash to delete a directory
-	dir := getDir(workPlacement) + "/"
-	if err := writer.RemoveObject(dir); err != nil {
-		logger.Error(err, "Error removing workloads from repository", "dir", dir)
-		return err
+func ignoreNotFound(err error) error {
+	if errors.Is(err, writers.FileNotFound) {
+		return nil
 	}
-	return nil
+	return err
+}
+
+func workloadsFilenames(works []v1alpha1.Workload) []string {
+	var result []string
+	for _, w := range works {
+		result = append(result, w.Filepath)
+	}
+	return result
+}
+
+func cleanupWorkloads(old []string, new []v1alpha1.Workload) []string {
+	works := make(map[string]bool)
+	for _, w := range new {
+		works[w.Filepath] = true
+	}
+	var result []string
+	for _, w := range old {
+		if _, ok := works[w]; !ok {
+			result = append(result, w)
+		}
+	}
+	return result
 }
 
 func getDir(workPlacement v1alpha1.WorkPlacement) string {