feat: Add support for writing to top-level dir of statestore (#130)

Introduces `.spec.filepath.mode` to the destination spec

Co-authored-by: Abby Bangser <abby@syntasso.io>
Co-authored-by: Chunyi Lyu <chunyi@syntasso.io>

api/v1alpha1/destination_types.go

@@ -58,6 +58,34 @@ type DestinationSpec struct {
 	// to this destination, unless the destination label set is also empty
 	// +kubebuilder:validation:Optional
 	StrictMatchLabels bool `json:"strictMatchLabels,omitempty"`
+
+	//The filepath mode to use when writing files to the destination.
+	Filepath Filepath `json:"filepath,omitempty"`
+}
+
+const (
+	//if modifying these dont forget to edit below where they are written as a
+	//kubebuilder comment for setting the default and Enum values.
+	FilepathExpressionTypeNone             = "none"
+	FilepathExpressionTypeNestedByMetadata = "nestedByMetadata"
+)
+
+type Filepath struct {
+	//+kubebuilder:default:=nestedByMetadata
+	//+kubebuilder:validation:Enum:={nestedByMetadata,none}
+	//The type of filepathExpression, either:
+	// - nestedByMetadata (default): files from the pipeline will be placed in a nested directory structure
+	// - none: file from the pipeline will be placed in a flat directory structure
+	Mode string `json:"type"`
+}
+
+// it gets defaulted by the K8s API, but for unit testing it wont be defaulted
+// since its not a real k8s api, so it may be empty when running unit tests.
+func (d *Destination) GetFilepathExpressionType() string {
+	if d.Spec.Filepath.Mode == "" {
+		return FilepathExpressionTypeNestedByMetadata
+	}
+	return d.Spec.Filepath.Mode
 }
 
 // DestinationStatus defines the observed state of Destination

config/crd/bases/platform.kratix.io_destinations.yaml

@@ -41,6 +41,22 @@ spec:
           spec:
             description: DestinationSpec defines the desired state of Destination
             properties:
+              filepath:
+                description: The filepath mode to use when writing files to the destination.
+                properties:
+                  type:
+                    default: nestedByMetadata
+                    description: |-
+                      The type of filepathExpression, either:
+                      - nestedByMetadata (default): files from the pipeline will be placed in a nested directory structure
+                      - none: file from the pipeline will be placed in a flat directory structure
+                    enum:
+                    - nestedByMetadata
+                    - none
+                    type: string
+                required:
+                - type
+                type: object
               path:
                 description: |-
                   Path within the StateStore to write documents. This path should be allocated

controllers/export_test.go

@@ -1,6 +1,11 @@
 package controllers
 
-import "github.com/syntasso/kratix/lib/workflow"
+import (
+	"github.com/go-logr/logr"
+	"github.com/syntasso/kratix/api/v1alpha1"
+	"github.com/syntasso/kratix/lib/workflow"
+	"github.com/syntasso/kratix/lib/writers"
+)
 
 func SetReconcileConfigureWorkflow(f func(workflow.Opts) (bool, error)) {
 	reconcileConfigure = f
@@ -9,3 +14,13 @@ func SetReconcileConfigureWorkflow(f func(workflow.Opts) (bool, error)) {
 func SetReconcileDeleteWorkflow(f func(workflow.Opts) (bool, error)) {
 	reconcileDelete = f
 }
+
+func SetNewS3Writer(f func(logger logr.Logger, stateStoreSpec v1alpha1.BucketStateStoreSpec, destination v1alpha1.Destination,
+	creds map[string][]byte) (writers.StateStoreWriter, error)) {
+	newS3Writer = f
+}
+
+func SetNewGitWriter(f func(logger logr.Logger, stateStoreSpec v1alpha1.GitStateStoreSpec, destination v1alpha1.Destination,
+	creds map[string][]byte) (writers.StateStoreWriter, error)) {
+	newGitWriter = f
+}

controllers/shared.go

@@ -28,6 +28,14 @@ const (
 	runDeleteWorkflowsFinalizer    = v1alpha1.KratixPrefix + "delete-workflows"
 )
 
+var (
+	newS3Writer func(logger logr.Logger, stateStoreSpec v1alpha1.BucketStateStoreSpec, destination v1alpha1.Destination,
+		creds map[string][]byte) (writers.StateStoreWriter, error) = writers.NewS3Writer
+
+	newGitWriter func(logger logr.Logger, stateStoreSpec v1alpha1.GitStateStoreSpec, destination v1alpha1.Destination,
+		creds map[string][]byte) (writers.StateStoreWriter, error) = writers.NewGitWriter
+)
+
 type StateStore interface {
 	client.Object
 	GetSecretRef() *v1.SecretReference
@@ -128,21 +136,20 @@ func newWriter(o opts, destination v1alpha1.Destination) (writers.StateStoreWrit
 			data = secret.Data
 		}
 
-		writer, err = writers.NewS3Writer(o.logger.WithName("writers").WithName("BucketStateStoreWriter"), stateStore.Spec, destination, data)
+		writer, err = newS3Writer(o.logger.WithName("writers").WithName("BucketStateStoreWriter"), stateStore.Spec, destination, data)
 	case "GitStateStore":
 		stateStore := &v1alpha1.GitStateStore{}
 		secret, fetchErr := fetchObjectAndSecret(o, stateStoreRef, stateStore)
 		if fetchErr != nil {
 			return nil, fetchErr
 		}
 
-		writer, err = writers.NewGitWriter(o.logger.WithName("writers").WithName("GitStateStoreWriter"), stateStore.Spec, destination, secret.Data)
+		writer, err = newGitWriter(o.logger.WithName("writers").WithName("GitStateStoreWriter"), stateStore.Spec, destination, secret.Data)
 	default:
 		return nil, fmt.Errorf("unsupported kind %s", destination.Spec.StateStoreRef.Kind)
 	}
 
 	if err != nil {
-		//TODO: should this be a retryable error?
 		o.logger.Error(err, "unable to create StateStoreWriter")
 		return nil, err
 	}

controllers/workplacement_controller.go

@@ -84,6 +84,7 @@ func (r *WorkPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		logger: logger,
 	}
 
+	//Mock this out
 	writer, err := newWriter(opts, *destination)
 	if err != nil {
 		if errors.IsNotFound(err) {
@@ -100,7 +101,7 @@ func (r *WorkPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return addFinalizers(opts, workPlacement, workPlacementFinalizers)
 	}
 
-	err = r.writeWorkloadsToStateStore(writer, *workPlacement, logger)
+	err = r.writeWorkloadsToStateStore(writer, *workPlacement, *destination, logger)
 	if err != nil {
 		logger.Error(err, "Error writing to repository, will try again in 5 seconds")
 		return defaultRequeue, err
@@ -129,8 +130,12 @@ func (r *WorkPlacementReconciler) deleteWorkPlacement(ctx context.Context, write
 	return fastRequeue, nil
 }
 
-func (r *WorkPlacementReconciler) writeWorkloadsToStateStore(writer writers.StateStoreWriter, workPlacement v1alpha1.WorkPlacement, logger logr.Logger) error {
-	err := writer.WriteDirWithObjects(writers.DeleteExistingContentsInDir, getDir(workPlacement), workPlacement.Spec.Workloads...)
+func (r *WorkPlacementReconciler) writeWorkloadsToStateStore(writer writers.StateStoreWriter, workPlacement v1alpha1.WorkPlacement, destination v1alpha1.Destination, logger logr.Logger) error {
+	dir := getDir(workPlacement)
+	if destination.GetFilepathExpressionType() == v1alpha1.FilepathExpressionTypeNone {
+		dir = ""
+	}
+	err := writer.WriteDirWithObjects(writers.DeleteExistingContentsInDir, dir, workPlacement.Spec.Workloads...)
 	if err != nil {
 		logger.Error(err, "Error writing resources to repository")
 		return err