Initial backup implementation

README.md

@@ -1,6 +1,10 @@
 # Commodore Component: rotating-bucket-backup
 
-This is a [Commodore][commodore] Component for rotating-bucket-backup.
+This is a [Commodore][commodore] Component to manage S3 bucket backups.
+
+The component creates a backup of an S3 bucket and stores it in another S3 bucket.
+The component creates a new bucket for each day of the month and stores the backup in the corresponding bucket.
+It then rotates by overwriting the corresponding bucket on the same day of the month.
 
 This repository is part of Project Syn.
 For documentation on Project Syn and this component, see [syn.tools](https://syn.tools).

class/defaults.yml

@@ -2,3 +2,14 @@ parameters:
   rotating_bucket_backup:
     =_metadata: {}
     namespace: syn-rotating-bucket-backup
+
+    images:
+      mc:
+        registry: quay.io
+        repository: minio/mc
+        tag: 'RELEASE.2024-09-09T07-53-10Z'
+
+    schedule:
+      hour: 1
+
+    jobs: {}

component/jobs.libsonnet

@@ -0,0 +1,113 @@
+// main template for rotating-bucket-backup
+local kap = import 'lib/kapitan.libjsonnet';
+local kube = import 'lib/kube.libjsonnet';
+local inv = kap.inventory();
+// The hiera parameters for the component
+local params = inv.parameters.rotating_bucket_backup;
+
+// one for each day in a month
+local bucketsPerTarget = 31;
+
+local scriptCM = kube.ConfigMap('rotating-bucket-backup') {
+  data: {
+    'backup.sh': (importstr 'scripts/backup.sh'),
+  },
+};
+
+local jobs = std.flattenArrays(std.mapWithIndex(function(jobI, job)
+  [
+    local bucketName = 'backup-%s-%d' % [ job, i ];
+    local jobParams = params.jobs[job];
+    assert jobParams.target_bucket_tmpl.type == 'appcat' : 'Currently only buckets with type `appcat` are supported';
+    kube._Object('appcat.vshn.io/v1', 'ObjectBucket', bucketName) {
+      spec+: {
+        parameters: {
+          bucketName: bucketName,
+          region: jobParams.target_bucket_tmpl.parameters.region,
+        },
+        writeConnectionSecretToRef: {
+          name: bucketName + '-bucket-secret',
+        },
+      },
+    }
+    for i in std.range(1, bucketsPerTarget)
+  ] + [
+    local jobParams = params.jobs[job];
+    kube.Secret('%s-source' % job) {
+      stringData+: {
+        SOURCE_URL: jobParams.source_bucket.url,
+        SOURCE_ACCESSKEY: jobParams.source_bucket.accesskey,
+        SOURCE_SECRETKEY: jobParams.source_bucket.secretkey,
+        SOURCE_BUCKET: jobParams.source_bucket.name,
+      },
+    },
+    kube.CronJob(job) {
+      metadata: {
+        name: job,
+      },
+      spec: {
+        jobTemplate: {
+          spec: {
+            backoffLimit: 4,
+            template: {
+              spec: {
+                containers: [
+                  {
+                    args: [
+                      '-c',
+                      '/script/backup.sh',
+                    ],
+                    command: [
+                      '/bin/bash',
+                    ],
+                    envFrom: [
+                      {
+                        secretRef: {
+                          name: '%s-source' % job,
+                        },
+                      },
+                    ] + [
+                      {
+                        secretRef: {
+                          name: 'backup-%s-%d-bucket-secret' % [ job, i ],
+                        },
+                        prefix: 'BUCKET_%d_' % [ i ],
+                      }
+                      for i in std.range(1, bucketsPerTarget)
+                    ],
+                    image: '%(registry)s/%(repository)s:%(tag)s' % params.images.mc,
+                    name: 'backup',
+                    resources: {},
+                    volumeMounts: [
+                      {
+                        mountPath: '/script',
+                        name: 'script',
+                      },
+                    ],
+                  },
+                ],
+                restartPolicy: 'Never',
+                terminationGracePeriodSeconds: 30,
+                volumes: [
+                  {
+                    configMap: {
+                      defaultMode: 511,
+                      name: scriptCM.metadata.name,
+                    },
+                    name: 'script',
+                  },
+                ],
+              },
+            },
+          },
+        },
+        schedule: '%d %d * * *' % [ (((jobI + 1) * 7) % 60), params.schedule.hour ],
+        successfulJobsHistoryLimit: 3,
+        failedJobsHistoryLimit: 3,
+        suspend: false,
+      },
+    },
+  ], std.objectFields(params.jobs)));
+
+
+jobs + [ scriptCM ]

component/main.jsonnet

@@ -7,4 +7,6 @@ local params = inv.parameters.rotating_bucket_backup;
 
 // Define outputs below
 {
+  '00_namespace': kube.Namespace(params.namespace),
+  '10_jobs': (import 'jobs.libsonnet'),
 }

component/scripts/backup.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eu
+
+day=$(date +%d)
+
+echo "It is day ${day} of the month. My backup journey will never end!"
+
+# Check if Secrets are loaded
+test -z "${SOURCE_URL}" && echo "SOURCE_URL is not set!" && exit 1
+test -z "${SOURCE_ACCESSKEY}" && echo "SOURCE_ACCESSKEY is not set!" && exit 1
+test -z "${SOURCE_SECRETKEY}" && echo "SOURCE_SECRETKEY is not set!" && exit 1
+test -z "${SOURCE_BUCKET}" && echo "SOURCE_BUCKET is not set!" && exit 1
+
+var_destination_url="BUCKET_${day}_ENDPOINT_URL"
+test -z "${!var_destination_url}" && echo "${var_destination_url} is not set!" && exit 1
+var_destination_accesskey="BUCKET_${day}_AWS_ACCESS_KEY_ID"
+test -z "${!var_destination_accesskey}" && echo "${var_destination_accesskey} is not set!" && exit 1
+var_destination_secretkey="BUCKET_${day}_AWS_SECRET_ACCESS_KEY"
+test -z "${!var_destination_secretkey}" && echo "${var_destination_secretkey} is not set!" && exit 1
+
+# Set the destination Bucket based on the day
+var_destination_bucket="BUCKET_${day}_BUCKET_NAME"
+test -z "${!var_destination_bucket}" && echo "${var_destination_bucket} is not set!" && exit 1
+destination_bucket="${!var_destination_bucket}"
+
+# Configure Source and Destination Bucket for minio cli
+mc --config-dir /tmp/ alias set source "${SOURCE_URL}" "${SOURCE_ACCESSKEY}" "${SOURCE_SECRETKEY}" --api S3v4
+mc --config-dir /tmp/ alias set destination "${!var_destination_url}" "${!var_destination_accesskey}" "${!var_destination_secretkey}" --api S3v4
+
+# check if source bucket exists
+mc --config-dir /tmp/ ls source | grep -q "${SOURCE_BUCKET}" || ( echo "Bucket ${SOURCE_BUCKET} does not exists!" && exit 1)
+
+# check if destination bucket exists
+mc --config-dir /tmp/ ls destination | grep -q "${destination_bucket}" || ( echo "Bucket ${destination_bucket} does not exists!" && exit 1)
+
+# Mirror source bucket into the destination bucket
+echo "Mirror ${SOURCE_BUCKET} bucket to ${destination_bucket} bucket"
+mc --config-dir /tmp/ mirror "source/${SOURCE_BUCKET}" "destination/${destination_bucket}" --overwrite --remove

docs/modules/ROOT/pages/index.adoc

@@ -1,5 +1,9 @@
 = rotating-bucket-backup
 
-rotating-bucket-backup is a Commodore component to manage rotating-bucket-backup.
+rotating-bucket-backup is a Commodore component to manage S3 bucket backups.
+
+The component creates a backup of an S3 bucket and stores it in another S3 bucket.
+The component creates a new bucket for each day of the month and stores the backup in the corresponding bucket.
+It then rotates by overwriting the corresponding bucket on the same day of the month.
 
 See the xref:references/parameters.adoc[parameters] reference for further details.

docs/modules/ROOT/pages/references/parameters.adoc

@@ -11,6 +11,87 @@ default:: `syn-rotating-bucket-backup`
 The namespace in which to deploy this component.
 
 
+== `images`
+
+[horizontal]
+type:: dict
+
+The images to use for the component.
+
+
+== `jobs`
+
+[horizontal]
+type:: dict
+default:: `{}`
+example::
++
+[source,yaml]
+----
+    jobs:
+      myjob:
+        source_bucket:
+          name: mytestbucket
+          accesskey: accesskey
+          secretkey: secretkey
+          url: https://objects.rma.example.com
+        target_bucket_tmpl:
+          type: appcat
+          parameters:
+            region: lpg
+----
+
+The backup jobs to run. The key is the name of the job. The value is a dictionary with the following keys:
+
+
+=== `source_bucket.name`
+
+[horizontal]
+type:: string
+
+The name of the source bucket.
+
+
+=== `source_bucket.accesskey`
+
+[horizontal]
+type:: string
+
+The access key for the source bucket.
+
+
+=== `source_bucket.secretkey`
+
+[horizontal]
+type:: string
+
+The secret key for the source bucket.
+
+
+=== `source_bucket.url`
+
+[horizontal]
+type:: string
+
+The URL of the source bucket.
+
+
+=== `target_bucket_tmpl.type`
+
+[horizontal]
+type:: string
+
+The type of the target bucket. Currently, only `appcat` is supported.
+
+
+=== `target_bucket_tmpl.parameters`
+
+[horizontal]
+type:: dict
+
+The parameters for the target bucket. The keys and values depend on the type of the target bucket.
+
+
 == Example
 
 [source,yaml]

tests/defaults.yml

@@ -1,3 +1,15 @@
 # Overwrite parameters here
 
-# parameters: {...}
+parameters:
+  rotating_bucket_backup:
+    jobs:
+      myjob:
+        source_bucket:
+          name: mytestbucket
+          accesskey: accesskey
+          secretkey: secretkey
+          url: https://objects.rma.example.com
+        target_bucket_tmpl:
+          type: appcat
+          parameters:
+            region: lpg

tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/00_namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations: {}
+  labels:
+    name: syn-rotating-bucket-backup
+  name: syn-rotating-bucket-backup