From f8cb886723f30386681635687e7e1ff10fec4e55 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Fri, 13 Sep 2024 16:53:29 +0200 Subject: [PATCH 1/4] Initial backup implementation --- README.md | 6 +- class/defaults.yml | 11 + component/jobs.libsonnet | 113 ++++ component/main.jsonnet | 2 + component/scripts/backup.sh | 39 ++ docs/modules/ROOT/pages/index.adoc | 6 +- .../ROOT/pages/references/parameters.adoc | 81 +++ tests/defaults.yml | 14 +- .../rotating-bucket-backup/00_namespace.yaml | 7 + .../rotating-bucket-backup/10_jobs.yaml | 627 ++++++++++++++++++ 10 files changed, 903 insertions(+), 3 deletions(-) create mode 100644 component/jobs.libsonnet create mode 100644 component/scripts/backup.sh create mode 100644 tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/00_namespace.yaml create mode 100644 tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml diff --git a/README.md b/README.md index 7fe3601..f12adfa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Commodore Component: rotating-bucket-backup -This is a [Commodore][commodore] Component for rotating-bucket-backup. +This is a [Commodore][commodore] Component to manage S3 bucket backups. + +The component creates a backup of an S3 bucket and stores it in another S3 bucket. +The component creates a new bucket for each day of the month and stores the backup in the corresponding bucket. +It then rotates by overwriting the corresponding bucket on the same day of the month. This repository is part of Project Syn. For documentation on Project Syn and this component, see [syn.tools](https://syn.tools). diff --git a/class/defaults.yml b/class/defaults.yml index e525676..dddc0a5 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -2,3 +2,14 @@ parameters: rotating_bucket_backup: =_metadata: {} namespace: syn-rotating-bucket-backup + + images: + mc: + registry: quay.io + repository: minio/mc + tag: 'RELEASE.2024-09-09T07-53-10Z' + + schedule: + hour: 1 + + jobs: {} diff --git a/component/jobs.libsonnet b/component/jobs.libsonnet new file mode 100644 index 0000000..8965a06 --- /dev/null +++ b/component/jobs.libsonnet @@ -0,0 +1,113 @@ +// main template for rotating-bucket-backup +local kap = import 'lib/kapitan.libjsonnet'; +local kube = import 'lib/kube.libjsonnet'; +local inv = kap.inventory(); +// The hiera parameters for the component +local params = inv.parameters.rotating_bucket_backup; + +// one for each day in a month +local bucketsPerTarget = 31; + +local scriptCM = kube.ConfigMap('rotating-bucket-backup') { + data: { + 'backup.sh': (importstr 'scripts/backup.sh'), + }, +}; + +local jobs = std.flattenArrays(std.mapWithIndex(function(jobI, job) + [ + local bucketName = 'backup-%s-%d' % [ job, i ]; + local jobParams = params.jobs[job]; + assert jobParams.target_bucket_tmpl.type == 'appcat' : 'Currently only buckets with type `appcat` are supported'; + kube._Object('appcat.vshn.io/v1', 'ObjectBucket', bucketName) { + spec+: { + parameters: { + bucketName: bucketName, + region: jobParams.target_bucket_tmpl.parameters.region, + }, + writeConnectionSecretToRef: { + name: bucketName + '-bucket-secret', + }, + }, + } + for i in std.range(1, bucketsPerTarget) + ] + [ + local jobParams = params.jobs[job]; + kube.Secret('%s-source' % job) { + stringData+: { + SOURCE_URL: jobParams.source_bucket.url, + SOURCE_ACCESSKEY: jobParams.source_bucket.accesskey, + SOURCE_SECRETKEY: jobParams.source_bucket.secretkey, + SOURCE_BUCKET: jobParams.source_bucket.name, + }, + }, + kube.CronJob(job) { + metadata: { + name: job, + }, + spec: { + jobTemplate: { + spec: { + backoffLimit: 4, + template: { + spec: { + containers: [ + { + args: [ + '-c', + '/script/backup.sh', + ], + command: [ + '/bin/bash', + ], + envFrom: [ + { + secretRef: { + name: '%s-source' % job, + }, + }, + ] + [ + { + secretRef: { + name: 'backup-%s-%d-bucket-secret' % [ job, i ], + }, + prefix: 'BUCKET_%d_' % [ i ], + } + for i in std.range(1, bucketsPerTarget) + ], + image: '%(registry)s/%(repository)s:%(tag)s' % params.images.mc, + name: 'backup', + resources: {}, + volumeMounts: [ + { + mountPath: '/script', + name: 'script', + }, + ], + }, + ], + restartPolicy: 'Never', + terminationGracePeriodSeconds: 30, + volumes: [ + { + configMap: { + defaultMode: 511, + name: scriptCM.metadata.name, + }, + name: 'script', + }, + ], + }, + }, + }, + }, + schedule: '%d %d * * *' % [ (((jobI + 1) * 7) % 60), params.schedule.hour ], + successfulJobsHistoryLimit: 3, + failedJobsHistoryLimit: 3, + suspend: false, + }, + }, + ], std.objectFields(params.jobs))); + + +jobs + [ scriptCM ] diff --git a/component/main.jsonnet b/component/main.jsonnet index 0b021b7..1ee1fc4 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -7,4 +7,6 @@ local params = inv.parameters.rotating_bucket_backup; // Define outputs below { + '00_namespace': kube.Namespace(params.namespace), + '10_jobs': (import 'jobs.libsonnet'), } diff --git a/component/scripts/backup.sh b/component/scripts/backup.sh new file mode 100644 index 0000000..84824c8 --- /dev/null +++ b/component/scripts/backup.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -eu + +day=$(date +%d) + +echo "It is day ${day} of the month. My backup journey will never end!" + +# Check if Secrets are loaded +test -z "${SOURCE_URL}" && echo "SOURCE_URL is not set!" && exit 1 +test -z "${SOURCE_ACCESSKEY}" && echo "SOURCE_ACCESSKEY is not set!" && exit 1 +test -z "${SOURCE_SECRETKEY}" && echo "SOURCE_SECRETKEY is not set!" && exit 1 +test -z "${SOURCE_BUCKET}" && echo "SOURCE_BUCKET is not set!" && exit 1 + +var_destination_url="BUCKET_${day}_ENDPOINT_URL" +test -z "${!var_destination_url}" && echo "${var_destination_url} is not set!" && exit 1 +var_destination_accesskey="BUCKET_${day}_AWS_ACCESS_KEY_ID" +test -z "${!var_destination_accesskey}" && echo "${var_destination_accesskey} is not set!" && exit 1 +var_destination_secretkey="BUCKET_${day}_AWS_SECRET_ACCESS_KEY" +test -z "${!var_destination_secretkey}" && echo "${var_destination_secretkey} is not set!" && exit 1 + +# Set the destination Bucket based on the day +var_destination_bucket="BUCKET_${day}_BUCKET_NAME" +test -z "${!var_destination_bucket}" && echo "${var_destination_bucket} is not set!" && exit 1 +destination_bucket="${!var_destination_bucket}" + +# Configure Source and Destination Bucket for minio cli +mc --config-dir /tmp/ alias set source "${SOURCE_URL}" "${SOURCE_ACCESSKEY}" "${SOURCE_SECRETKEY}" --api S3v4 +mc --config-dir /tmp/ alias set destination "${!var_destination_url}" "${!var_destination_accesskey}" "${!var_destination_secretkey}" --api S3v4 + +# check if source bucket exists +mc --config-dir /tmp/ ls source | grep -q "${SOURCE_BUCKET}" || ( echo "Bucket ${SOURCE_BUCKET} does not exists!" && exit 1) + +# check if destination bucket exists +mc --config-dir /tmp/ ls destination | grep -q "${destination_bucket}" || ( echo "Bucket ${destination_bucket} does not exists!" && exit 1) + +# Mirror source bucket into the destination bucket +echo "Mirror ${SOURCE_BUCKET} bucket to ${destination_bucket} bucket" +mc --config-dir /tmp/ mirror "source/${SOURCE_BUCKET}" "destination/${destination_bucket}" --overwrite --remove diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index c9d9eab..ab1d080 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -1,5 +1,9 @@ = rotating-bucket-backup -rotating-bucket-backup is a Commodore component to manage rotating-bucket-backup. +rotating-bucket-backup is a Commodore component to manage S3 bucket backups. + +The component creates a backup of an S3 bucket and stores it in another S3 bucket. +The component creates a new bucket for each day of the month and stores the backup in the corresponding bucket. +It then rotates by overwriting the corresponding bucket on the same day of the month. See the xref:references/parameters.adoc[parameters] reference for further details. diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index b0b58fe..9164a05 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -11,6 +11,87 @@ default:: `syn-rotating-bucket-backup` The namespace in which to deploy this component. +== `images` + +[horizontal] +type:: dict + +The images to use for the component. + + +== `jobs` + +[horizontal] +type:: dict +default:: `{}` +example:: ++ +[source,yaml] +---- + jobs: + myjob: + source_bucket: + name: mytestbucket + accesskey: accesskey + secretkey: secretkey + url: https://objects.rma.example.com + target_bucket_tmpl: + type: appcat + parameters: + region: lpg +---- + +The backup jobs to run. The key is the name of the job. The value is a dictionary with the following keys: + + +=== `source_bucket.name` + +[horizontal] +type:: string + +The name of the source bucket. + + +=== `source_bucket.accesskey` + +[horizontal] +type:: string + +The access key for the source bucket. + + +=== `source_bucket.secretkey` + +[horizontal] +type:: string + +The secret key for the source bucket. + + +=== `source_bucket.url` + +[horizontal] +type:: string + +The URL of the source bucket. + + +=== `target_bucket_tmpl.type` + +[horizontal] +type:: string + +The type of the target bucket. Currently, only `appcat` is supported. + + +=== `target_bucket_tmpl.parameters` + +[horizontal] +type:: dict + +The parameters for the target bucket. The keys and values depend on the type of the target bucket. + + == Example [source,yaml] diff --git a/tests/defaults.yml b/tests/defaults.yml index a4da5b7..7b241e1 100644 --- a/tests/defaults.yml +++ b/tests/defaults.yml @@ -1,3 +1,15 @@ # Overwrite parameters here -# parameters: {...} +parameters: + rotating_bucket_backup: + jobs: + myjob: + source_bucket: + name: mytestbucket + accesskey: accesskey + secretkey: secretkey + url: https://objects.rma.example.com + target_bucket_tmpl: + type: appcat + parameters: + region: lpg diff --git a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/00_namespace.yaml b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/00_namespace.yaml new file mode 100644 index 0000000..7249aa1 --- /dev/null +++ b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-rotating-bucket-backup + name: syn-rotating-bucket-backup diff --git a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml new file mode 100644 index 0000000..5f1785f --- /dev/null +++ b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml @@ -0,0 +1,627 @@ +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-1 + name: backup-myjob-1 +spec: + parameters: + bucketName: backup-myjob-1 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-1-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-2 + name: backup-myjob-2 +spec: + parameters: + bucketName: backup-myjob-2 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-2-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-3 + name: backup-myjob-3 +spec: + parameters: + bucketName: backup-myjob-3 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-3-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-4 + name: backup-myjob-4 +spec: + parameters: + bucketName: backup-myjob-4 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-4-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-5 + name: backup-myjob-5 +spec: + parameters: + bucketName: backup-myjob-5 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-5-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-6 + name: backup-myjob-6 +spec: + parameters: + bucketName: backup-myjob-6 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-6-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-7 + name: backup-myjob-7 +spec: + parameters: + bucketName: backup-myjob-7 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-7-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-8 + name: backup-myjob-8 +spec: + parameters: + bucketName: backup-myjob-8 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-8-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-9 + name: backup-myjob-9 +spec: + parameters: + bucketName: backup-myjob-9 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-9-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-10 + name: backup-myjob-10 +spec: + parameters: + bucketName: backup-myjob-10 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-10-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-11 + name: backup-myjob-11 +spec: + parameters: + bucketName: backup-myjob-11 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-11-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-12 + name: backup-myjob-12 +spec: + parameters: + bucketName: backup-myjob-12 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-12-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-13 + name: backup-myjob-13 +spec: + parameters: + bucketName: backup-myjob-13 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-13-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-14 + name: backup-myjob-14 +spec: + parameters: + bucketName: backup-myjob-14 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-14-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-15 + name: backup-myjob-15 +spec: + parameters: + bucketName: backup-myjob-15 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-15-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-16 + name: backup-myjob-16 +spec: + parameters: + bucketName: backup-myjob-16 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-16-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-17 + name: backup-myjob-17 +spec: + parameters: + bucketName: backup-myjob-17 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-17-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-18 + name: backup-myjob-18 +spec: + parameters: + bucketName: backup-myjob-18 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-18-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-19 + name: backup-myjob-19 +spec: + parameters: + bucketName: backup-myjob-19 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-19-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-20 + name: backup-myjob-20 +spec: + parameters: + bucketName: backup-myjob-20 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-20-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-21 + name: backup-myjob-21 +spec: + parameters: + bucketName: backup-myjob-21 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-21-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-22 + name: backup-myjob-22 +spec: + parameters: + bucketName: backup-myjob-22 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-22-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-23 + name: backup-myjob-23 +spec: + parameters: + bucketName: backup-myjob-23 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-23-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-24 + name: backup-myjob-24 +spec: + parameters: + bucketName: backup-myjob-24 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-24-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-25 + name: backup-myjob-25 +spec: + parameters: + bucketName: backup-myjob-25 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-25-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-26 + name: backup-myjob-26 +spec: + parameters: + bucketName: backup-myjob-26 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-26-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-27 + name: backup-myjob-27 +spec: + parameters: + bucketName: backup-myjob-27 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-27-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-28 + name: backup-myjob-28 +spec: + parameters: + bucketName: backup-myjob-28 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-28-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-29 + name: backup-myjob-29 +spec: + parameters: + bucketName: backup-myjob-29 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-29-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-30 + name: backup-myjob-30 +spec: + parameters: + bucketName: backup-myjob-30 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-30-bucket-secret +--- +apiVersion: appcat.vshn.io/v1 +kind: ObjectBucket +metadata: + annotations: {} + labels: + name: backup-myjob-31 + name: backup-myjob-31 +spec: + parameters: + bucketName: backup-myjob-31 + region: lpg + writeConnectionSecretToRef: + name: backup-myjob-31-bucket-secret +--- +apiVersion: v1 +data: {} +kind: Secret +metadata: + annotations: {} + labels: + name: myjob-source + name: myjob-source +stringData: + SOURCE_ACCESSKEY: accesskey + SOURCE_BUCKET: mytestbucket + SOURCE_SECRETKEY: secretkey + SOURCE_URL: https://objects.rma.example.com +type: Opaque +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: myjob +spec: + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 4 + template: + spec: + containers: + - args: + - -c + - /script/backup.sh + command: + - /bin/bash + envFrom: + - secretRef: + name: myjob-source + - prefix: BUCKET_1_ + secretRef: + name: backup-myjob-1-bucket-secret + - prefix: BUCKET_2_ + secretRef: + name: backup-myjob-2-bucket-secret + - prefix: BUCKET_3_ + secretRef: + name: backup-myjob-3-bucket-secret + - prefix: BUCKET_4_ + secretRef: + name: backup-myjob-4-bucket-secret + - prefix: BUCKET_5_ + secretRef: + name: backup-myjob-5-bucket-secret + - prefix: BUCKET_6_ + secretRef: + name: backup-myjob-6-bucket-secret + - prefix: BUCKET_7_ + secretRef: + name: backup-myjob-7-bucket-secret + - prefix: BUCKET_8_ + secretRef: + name: backup-myjob-8-bucket-secret + - prefix: BUCKET_9_ + secretRef: + name: backup-myjob-9-bucket-secret + - prefix: BUCKET_10_ + secretRef: + name: backup-myjob-10-bucket-secret + - prefix: BUCKET_11_ + secretRef: + name: backup-myjob-11-bucket-secret + - prefix: BUCKET_12_ + secretRef: + name: backup-myjob-12-bucket-secret + - prefix: BUCKET_13_ + secretRef: + name: backup-myjob-13-bucket-secret + - prefix: BUCKET_14_ + secretRef: + name: backup-myjob-14-bucket-secret + - prefix: BUCKET_15_ + secretRef: + name: backup-myjob-15-bucket-secret + - prefix: BUCKET_16_ + secretRef: + name: backup-myjob-16-bucket-secret + - prefix: BUCKET_17_ + secretRef: + name: backup-myjob-17-bucket-secret + - prefix: BUCKET_18_ + secretRef: + name: backup-myjob-18-bucket-secret + - prefix: BUCKET_19_ + secretRef: + name: backup-myjob-19-bucket-secret + - prefix: BUCKET_20_ + secretRef: + name: backup-myjob-20-bucket-secret + - prefix: BUCKET_21_ + secretRef: + name: backup-myjob-21-bucket-secret + - prefix: BUCKET_22_ + secretRef: + name: backup-myjob-22-bucket-secret + - prefix: BUCKET_23_ + secretRef: + name: backup-myjob-23-bucket-secret + - prefix: BUCKET_24_ + secretRef: + name: backup-myjob-24-bucket-secret + - prefix: BUCKET_25_ + secretRef: + name: backup-myjob-25-bucket-secret + - prefix: BUCKET_26_ + secretRef: + name: backup-myjob-26-bucket-secret + - prefix: BUCKET_27_ + secretRef: + name: backup-myjob-27-bucket-secret + - prefix: BUCKET_28_ + secretRef: + name: backup-myjob-28-bucket-secret + - prefix: BUCKET_29_ + secretRef: + name: backup-myjob-29-bucket-secret + - prefix: BUCKET_30_ + secretRef: + name: backup-myjob-30-bucket-secret + - prefix: BUCKET_31_ + secretRef: + name: backup-myjob-31-bucket-secret + image: quay.io/minio/mc:RELEASE.2024-09-09T07-53-10Z + name: backup + resources: {} + volumeMounts: + - mountPath: /script + name: script + restartPolicy: Never + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 511 + name: rotating-bucket-backup + name: script + schedule: 7 1 * * * + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: v1 +data: + backup.sh: | + #!/bin/bash + + set -eu + + day=$(date +%d) + + echo "It is day ${day} of the month. My backup journey will never end!" + + # Check if Secrets are loaded + test -z "${SOURCE_URL}" && echo "SOURCE_URL is not set!" && exit 1 + test -z "${SOURCE_ACCESSKEY}" && echo "SOURCE_ACCESSKEY is not set!" && exit 1 + test -z "${SOURCE_SECRETKEY}" && echo "SOURCE_SECRETKEY is not set!" && exit 1 + test -z "${SOURCE_BUCKET}" && echo "SOURCE_BUCKET is not set!" && exit 1 + + var_destination_url="BUCKET_${day}_ENDPOINT_URL" + test -z "${!var_destination_url}" && echo "${var_destination_url} is not set!" && exit 1 + var_destination_accesskey="BUCKET_${day}_AWS_ACCESS_KEY_ID" + test -z "${!var_destination_accesskey}" && echo "${var_destination_accesskey} is not set!" && exit 1 + var_destination_secretkey="BUCKET_${day}_AWS_SECRET_ACCESS_KEY" + test -z "${!var_destination_secretkey}" && echo "${var_destination_secretkey} is not set!" && exit 1 + + # Set the destination Bucket based on the day + var_destination_bucket="BUCKET_${day}_BUCKET_NAME" + test -z "${!var_destination_bucket}" && echo "${var_destination_bucket} is not set!" && exit 1 + destination_bucket="${!var_destination_bucket}" + + # Configure Source and Destination Bucket for minio cli + mc --config-dir /tmp/ alias set source "${SOURCE_URL}" "${SOURCE_ACCESSKEY}" "${SOURCE_SECRETKEY}" --api S3v4 + mc --config-dir /tmp/ alias set destination "${!var_destination_url}" "${!var_destination_accesskey}" "${!var_destination_secretkey}" --api S3v4 + + # check if source bucket exists + mc --config-dir /tmp/ ls source | grep -q "${SOURCE_BUCKET}" || ( echo "Bucket ${SOURCE_BUCKET} does not exists!" && exit 1) + + # check if destination bucket exists + mc --config-dir /tmp/ ls destination | grep -q "${destination_bucket}" || ( echo "Bucket ${destination_bucket} does not exists!" && exit 1) + + # Mirror source bucket into the destination bucket + echo "Mirror ${SOURCE_BUCKET} bucket to ${destination_bucket} bucket" + mc --config-dir /tmp/ mirror "source/${SOURCE_BUCKET}" "destination/${destination_bucket}" --overwrite --remove +kind: ConfigMap +metadata: + annotations: {} + labels: + name: rotating-bucket-backup + name: rotating-bucket-backup From 79077683bcfa3ae65f3b10dec44202b6f423ee22 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 17 Sep 2024 11:05:08 +0200 Subject: [PATCH 2/4] Fix sync loop with appcat/argocd https://docs.appcat.ch/references/argocd.html --- component/jobs.libsonnet | 6 + .../rotating-bucket-backup/10_jobs.yaml | 124 +++++++++++++----- 2 files changed, 99 insertions(+), 31 deletions(-) diff --git a/component/jobs.libsonnet b/component/jobs.libsonnet index 8965a06..7b4dcb8 100644 --- a/component/jobs.libsonnet +++ b/component/jobs.libsonnet @@ -20,6 +20,12 @@ local jobs = std.flattenArrays(std.mapWithIndex(function(jobI, job) local jobParams = params.jobs[job]; assert jobParams.target_bucket_tmpl.type == 'appcat' : 'Currently only buckets with type `appcat` are supported'; kube._Object('appcat.vshn.io/v1', 'ObjectBucket', bucketName) { + metadata+: { + annotations: { + 'argocd.argoproj.io/compare-options': 'IgnoreExtraneous', + 'argocd.argoproj.io/sync-options': 'Prune=false', + }, + }, spec+: { parameters: { bucketName: bucketName, diff --git a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml index 5f1785f..f4f4051 100644 --- a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml +++ b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml @@ -1,7 +1,9 @@ apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-1 name: backup-myjob-1 @@ -15,7 +17,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-2 name: backup-myjob-2 @@ -29,7 +33,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-3 name: backup-myjob-3 @@ -43,7 +49,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-4 name: backup-myjob-4 @@ -57,7 +65,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-5 name: backup-myjob-5 @@ -71,7 +81,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-6 name: backup-myjob-6 @@ -85,7 +97,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-7 name: backup-myjob-7 @@ -99,7 +113,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-8 name: backup-myjob-8 @@ -113,7 +129,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-9 name: backup-myjob-9 @@ -127,7 +145,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-10 name: backup-myjob-10 @@ -141,7 +161,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-11 name: backup-myjob-11 @@ -155,7 +177,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-12 name: backup-myjob-12 @@ -169,7 +193,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-13 name: backup-myjob-13 @@ -183,7 +209,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-14 name: backup-myjob-14 @@ -197,7 +225,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-15 name: backup-myjob-15 @@ -211,7 +241,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-16 name: backup-myjob-16 @@ -225,7 +257,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-17 name: backup-myjob-17 @@ -239,7 +273,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-18 name: backup-myjob-18 @@ -253,7 +289,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-19 name: backup-myjob-19 @@ -267,7 +305,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-20 name: backup-myjob-20 @@ -281,7 +321,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-21 name: backup-myjob-21 @@ -295,7 +337,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-22 name: backup-myjob-22 @@ -309,7 +353,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-23 name: backup-myjob-23 @@ -323,7 +369,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-24 name: backup-myjob-24 @@ -337,7 +385,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-25 name: backup-myjob-25 @@ -351,7 +401,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-26 name: backup-myjob-26 @@ -365,7 +417,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-27 name: backup-myjob-27 @@ -379,7 +433,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-28 name: backup-myjob-28 @@ -393,7 +449,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-29 name: backup-myjob-29 @@ -407,7 +465,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-30 name: backup-myjob-30 @@ -421,7 +481,9 @@ spec: apiVersion: appcat.vshn.io/v1 kind: ObjectBucket metadata: - annotations: {} + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false labels: name: backup-myjob-31 name: backup-myjob-31 From 0a773404afe06a29ffe15c5d35b202a36f751c02 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 17 Sep 2024 13:49:01 +0200 Subject: [PATCH 3/4] Use image version with grep --- class/defaults.yml | 3 ++- .../rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index dddc0a5..e58e837 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -7,7 +7,8 @@ parameters: mc: registry: quay.io repository: minio/mc - tag: 'RELEASE.2024-09-09T07-53-10Z' + # note: newer release are based on RHEL9 containers which do not include grep + tag: 'RELEASE.2022-11-07T23-47-39Z' schedule: hour: 1 diff --git a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml index f4f4051..db49d67 100644 --- a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml +++ b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/10_jobs.yaml @@ -622,7 +622,7 @@ spec: - prefix: BUCKET_31_ secretRef: name: backup-myjob-31-bucket-secret - image: quay.io/minio/mc:RELEASE.2024-09-09T07-53-10Z + image: quay.io/minio/mc:RELEASE.2022-11-07T23-47-39Z name: backup resources: {} volumeMounts: From 0acecc1dcfea5a934623bb985992e9256ebe8f62 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 17 Sep 2024 17:40:20 +0200 Subject: [PATCH 4/4] Add alerting --- class/defaults.yml | 28 ++++++++ component/main.jsonnet | 35 +++++++++- .../ROOT/pages/references/parameters.adoc | 69 +++++++++++++++++++ .../20_prometheus_rule.yaml | 35 ++++++++++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/20_prometheus_rule.yaml diff --git a/class/defaults.yml b/class/defaults.yml index e58e837..20ff99d 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -2,6 +2,9 @@ parameters: rotating_bucket_backup: =_metadata: {} namespace: syn-rotating-bucket-backup + namespace_metadata: + annotations: {} + labels: {} images: mc: @@ -14,3 +17,28 @@ parameters: hour: 1 jobs: {} + + monitoring: + enabled: true + alerts: + RotatingClusterBackup_JobFailed: + enabled: true + rule: + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert. + expr: | + kube_job_failed{namespace="${rotating_bucket_backup:namespace}"} > 0 + for: 15m + labels: + severity: warning + + RotatingClusterBackup_JobCompletion_12h: + enabled: true + rule: + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete. Job did not complete in time + expr: | + kube_job_spec_completions{namespace="${rotating_bucket_backup:namespace}"} - kube_job_status_succeeded{namespace="${rotating_bucket_backup:namespace}"} > 0 + for: 12h + labels: + severity: warning diff --git a/component/main.jsonnet b/component/main.jsonnet index 1ee1fc4..6e7ea51 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -1,12 +1,45 @@ // main template for rotating-bucket-backup +local com = import 'lib/commodore.libjsonnet'; local kap = import 'lib/kapitan.libjsonnet'; local kube = import 'lib/kube.libjsonnet'; local inv = kap.inventory(); // The hiera parameters for the component local params = inv.parameters.rotating_bucket_backup; +local alertlabels = { + syn: 'true', + syn_component: 'rotating-bucket-backup', +}; + +local alerts = params.monitoring.alerts; +local prometheusRule = + kube._Object('monitoring.coreos.com/v1', 'PrometheusRule', 'rotating-bucket-backup-alerts') { + metadata+: { + namespace: params.namespace, + }, + spec+: { + groups+: [ + { + name: 'rotating-bucket-backup-alerts', + rules: + std.filterMap( + function(field) alerts[field].enabled == true, + function(field) alerts[field].rule { + alert: field, + labels+: alertlabels, + }, + std.sort(std.objectFields(alerts)), + ), + }, + ], + }, + }; + // Define outputs below { - '00_namespace': kube.Namespace(params.namespace), + '00_namespace': kube.Namespace(params.namespace) { + metadata+: com.makeMergeable(params.namespace_metadata), + }, '10_jobs': (import 'jobs.libsonnet'), + [if params.monitoring.enabled then '20_prometheus_rule']: prometheusRule, } diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 9164a05..c2eba7a 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -11,6 +11,21 @@ default:: `syn-rotating-bucket-backup` The namespace in which to deploy this component. +== `namespace_metadata` + +[horizontal] +type:: dict +default:: ++ +[source,yaml] +---- +annotations: {} +labels: {} +---- + +Allows adding additional annotations and labels to the namespace. + + == `images` [horizontal] @@ -92,6 +107,60 @@ type:: dict The parameters for the target bucket. The keys and values depend on the type of the target bucket. +== `monitoring.enabled` + +[horizontal] +type:: boolean +default:: `true` + +Controls if `PrometheusRule` objects are created. +The alerts expect `kube-state-metrics` to be running in the cluster. + + +== `monitoring.alerts` + +[horizontal] +type:: dict +default:: See https://github.com/vshn/component-rotating-bucket-backup/blob/master/class/defaults.yml[defaults.yml] +example:: ++ +[source,yaml] +---- +alerts: + MyAlert: + enabled: true + rule: + expr: | + up{namespace="${rotating_bucket_backup:namespace}"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." +---- + +The alerts to configure. +The key is the name of the alert. +The value is a dictionary with the following keys: + +=== `enabled` + +[horizontal] +type:: boolean + +Controls if the alert is enabled. + + +=== `rule` + +[horizontal] +type:: dict + +The Prometheus rule for the alert. +The keys and values depend on the alert. + + == Example [source,yaml] diff --git a/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/20_prometheus_rule.yaml b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/20_prometheus_rule.yaml new file mode 100644 index 0000000..d87e8c5 --- /dev/null +++ b/tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/20_prometheus_rule.yaml @@ -0,0 +1,35 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + annotations: {} + labels: + name: rotating-bucket-backup-alerts + name: rotating-bucket-backup-alerts + namespace: syn-rotating-bucket-backup +spec: + groups: + - name: rotating-bucket-backup-alerts + rules: + - alert: RotatingClusterBackup_JobCompletion_12h + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking + more than 12 hours to complete. Job did not complete in time + expr: | + kube_job_spec_completions{namespace="syn-rotating-bucket-backup"} - kube_job_status_succeeded{namespace="syn-rotating-bucket-backup"} > 0 + for: 12h + labels: + severity: warning + syn: 'true' + syn_component: rotating-bucket-backup + - alert: RotatingClusterBackup_JobFailed + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed + to complete. Removing failed job after investigation should clear this + alert. + expr: | + kube_job_failed{namespace="syn-rotating-bucket-backup"} > 0 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: rotating-bucket-backup