From 197c03576a3a9da93f3ccc892137ff9838bb6b62 Mon Sep 17 00:00:00 2001 From: Aline Abler Date: Mon, 15 Jan 2024 11:06:14 +0100 Subject: [PATCH] Add queries for APPUiO Cloud (#14) --- class/defaults.yml | 232 +++ .../appuio-reporting/11_backfill.yaml | 1324 ++++++++++++++++ .../appuio-reporting/11_backfill.yaml | 1405 +++++++++++++++++ .../appuio-reporting/11_backfill.yaml | 1324 ++++++++++++++++ .../appuio-reporting/11_backfill.yaml | 1333 ++++++++++++++++ .../appuio-reporting/11_backfill.yaml | 1324 ++++++++++++++++ .../appuio-reporting/11_backfill.yaml | 1324 ++++++++++++++++ 7 files changed, 8266 insertions(+) diff --git a/class/defaults.yml b/class/defaults.yml index f1d0448..64042e2 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -589,6 +589,238 @@ parameters: vshn_service_level="%(vshn_service_level)s", } + appuio_cloud_compute: + enabled: true + prometheus_org_id: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + products: + - product_id: appuiocloud-compute-cloudscaleflex + params: + node_class: flex + zone: c-appuio-cloudscale-lpg-2 + cpu_ratio: "4294967296" # 4096 MiB / core + - product_id: appuiocloud-compute-cloudscaleplus + params: + node_class: plus + zone: c-appuio-cloudscale-lpg-2 + cpu_ratio: "4294967296" # 4096 MiB / core + - product_id: appuiocloud-compute-exoscalestandard + params: + node_class: "" + zone: c-appuio-exoscale-ch-gva-2-0 + cpu_ratio: "5333057536" # 5086 MiB / core + instance_id_pattern: '%(cluster_id)s-%(namespace)s' + item_group_description_pattern: 'APPUiO Cloud - Zone: %(cluster_id)s / Namespace: %(namespace)s' + unit_id: uom_uom_180_8459f204 # MiB - Minute + query_pattern: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="%(zone)s"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="%(node_class)s"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="%(zone)s"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="%(node_class)s"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="%(zone)s"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="%(node_class)s"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="%(zone)s"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="%(node_class)s"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="%(zone)s"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="%(node_class)s"} + # CPU ratio depending on cluster + * %(cpu_ratio)s + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="%(zone)s"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="%(node_class)s"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + + appuio_cloud_storage: + enabled: true + prometheus_org_id: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + products: + - product_id: appuiocloud-blockstorage-cloudscalebulk + params: + storage_class: bulk.* + zone: c-appuio-cloudscale-lpg-2 + - product_id: appuiocloud-blockstorage-cloudscalessd + params: + storage_class: ssd.* + zone: c-appuio-cloudscale-lpg-2 + - product_id: appuiocloud-filestorage-cloudscalessd + params: + storage_class: cephfs-fspool-cluster + zone: c-appuio-cloudscale-lpg-2 + - product_id: appuiocloud-blockstorage-exoscalessd + params: + storage_class: ssd.* + zone: c-appuio-exoscale-ch-gva-2-0 + - product_id: appuiocloud-filestorage-exoscalessd + params: + storage_class: cephfs-fspool-cluster + zone: c-appuio-exoscale-ch-gva-2-0 + instance_id_pattern: '%(cluster_id)s-%(namespace)s' + item_group_description_pattern: 'APPUiO Cloud - Zone: %(cluster_id)s / Namespace: %(namespace)s' + unit_id: uom_uom_180_8459f204 # MiB - Minute + query_pattern: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="%(zone)s",storageclass=~"%(storage_class)s"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + + appuio_cloud_loadbalancer: + enabled: true + prometheus_org_id: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + products: + - product_id: appuiocloud-loadbalancer-cloudscale + params: + zone: c-appuio-cloudscale-lpg-2 + instance_id_pattern: '%(cluster_id)s-%(namespace)s' + item_group_description_pattern: 'APPUiO Cloud - Zone: %(cluster_id)s / Namespace: %(namespace)s' + unit_id: uom_uom_66_24bffda7 # service-minutes + query_pattern: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="%(zone)s"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) network_policies: target_namespaces: {} diff --git a/tests/golden/defaults/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/defaults/appuio-reporting/appuio-reporting/11_backfill.yaml index 483d17a..2d42749 100644 --- a/tests/golden/defaults/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/defaults/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1329 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: openshift-cloudscale-storagevcpu-besteffort diff --git a/tests/golden/external-secret/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/external-secret/appuio-reporting/appuio-reporting/11_backfill.yaml index d8ccd85..e60b835 100644 --- a/tests/golden/external-secret/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/external-secret/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1410 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + volumeMounts: + - mountPath: /secrets/database + name: dbsecret + readOnly: true + initContainers: [] + restartPolicy: OnFailure + volumes: + - name: dbsecret + secret: + defaultMode: 384 + secretName: reporting-db-prod-cred + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: openshift-cloudscale-storagevcpu-besteffort diff --git a/tests/golden/suspended-cronjobs/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/suspended-cronjobs/appuio-reporting/appuio-reporting/11_backfill.yaml index a0a79d8..8d28def 100644 --- a/tests/golden/suspended-cronjobs/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/suspended-cronjobs/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1329 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: true +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: openshift-cloudscale-storagevcpu-besteffort diff --git a/tests/golden/test-mode/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/test-mode/appuio-reporting/appuio-reporting/11_backfill.yaml index 26ae51b..2e2cc2b 100644 --- a/tests/golden/test-mode/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/test-mode/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1338 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + --debug-override-sales-order-id=OVERRIDE + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: openshift-cloudscale-storagevcpu-besteffort diff --git a/tests/golden/with-org-id/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/with-org-id/appuio-reporting/appuio-reporting/11_backfill.yaml index 47126f8..4bc52ca 100644 --- a/tests/golden/with-org-id/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/with-org-id/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1329 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: openshift-cloudscale-storagevcpu-besteffort diff --git a/tests/golden/with-rules/appuio-reporting/appuio-reporting/11_backfill.yaml b/tests/golden/with-rules/appuio-reporting/appuio-reporting/11_backfill.yaml index 2f3b6b7..bbb33ca 100644 --- a/tests/golden/with-rules/appuio-reporting/appuio-reporting/11_backfill.yaml +++ b/tests/golden/with-rules/appuio-reporting/appuio-reporting/11_backfill.yaml @@ -1,5 +1,1329 @@ apiVersion: batch/v1 kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleflex-0 + name: appuiocloud-compute-cloudscaleflex-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleflex + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleflex-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleflex + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="flex"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="flex"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-cloudscaleplus-1 + name: appuiocloud-compute-cloudscaleplus-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-cloudscaleplus + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-cloudscaleplus-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-cloudscaleplus + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class="plus"})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-cloudscale-lpg-2"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"} + # CPU ratio depending on cluster + * 4294967296 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-cloudscale-lpg-2"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class="plus"}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-compute-exoscalestandard-2 + name: appuiocloud-compute-exoscalestandard-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-compute-exoscalestandard + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-compute-exoscalestandard-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-compute-exoscalestandard + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + # Average over a one-minute time frame. + # NOTE: This is a sliding window. Results vary based on the queries execution time. + avg_over_time( + clamp_min( + ( + # Get the maximum of requested and used memory. + # TODO Is there a better way to get the maximum of two vectors? + ( + # Select used memory if higher. + ( + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped. + >= + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + ) + or + # Select reserved memory if higher. + ( + # IMPORTANT: The desired time series must always be first. + sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class=""})) + > + sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!="", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + ) + ) + # Add CPU requests in violation to the ratio provided by the platform. + + clamp_min( + # Convert CPU request to their memory equivalent. + sum by(cluster_id, namespace, label_appuio_io_node_class) ( + kube_pod_container_resource_requests{resource="cpu", cluster_id="c-appuio-exoscale-ch-gva-2-0"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""} + # CPU ratio depending on cluster + * 5333057536 + ) + # Subtract memory request + - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory", cluster_id="c-appuio-exoscale-ch-gva-2-0"} + * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} + * on(cluster_id, node) group_left(label_appuio_io_node_class) kube_node_labels{label_appuio_io_node_class=""}) + # Only values above zero are in violation. + , 0) + ) + * + # Join namespace label `label_appuio_io_organization` as `organization`. + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ), + # At least return 128MiB + 128 * 1024 * 1024 + )[45s:15s] + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-loadbalancer-cloudscale-0 + name: appuiocloud-loadbalancer-cloudscale-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-loadbalancer-cloudscale + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-loadbalancer-cloudscale-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-loadbalancer-cloudscale + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + ( + # Get number of services of type load balancer + sum by(cluster_id, namespace) (kube_service_spec_type{type="LoadBalancer", cluster_id="c-appuio-cloudscale-lpg-2"}) + * + # Join the namespace label to get the tenant + on(cluster_id, namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + * + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + )[59m:1m] + ) + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_66_24bffda7 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalebulk-0 + name: appuiocloud-blockstorage-cloudscalebulk-0 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalebulk + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalebulk-0 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalebulk + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"bulk.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-cloudscalessd-1 + name: appuiocloud-blockstorage-cloudscalessd-1 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-cloudscalessd-1 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-cloudscalessd-2 + name: appuiocloud-filestorage-cloudscalessd-2 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-cloudscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-cloudscalessd-2 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-cloudscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-cloudscale-lpg-2",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-blockstorage-exoscalessd-3 + name: appuiocloud-blockstorage-exoscalessd-3 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-blockstorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-blockstorage-exoscalessd-3 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-blockstorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"ssd.*"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + name: appuiocloud-filestorage-exoscalessd-4 + name: appuiocloud-filestorage-exoscalessd-4 + namespace: appuio-reporting +spec: + failedJobsHistoryLimit: 768 + jobTemplate: + metadata: + annotations: + product-id: appuiocloud-filestorage-exoscalessd + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + cron-job-name: appuiocloud-filestorage-exoscalessd-4 + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: appuio-reporting + app.kubernetes.io/part-of: syn + spec: + containers: + - args: + - appuio-reporting report --timerange 1h --begin=$(date -d "now -3 + hours" -u +"%Y-%m-%dT%H:00:00Z") --repeat-until=$(date -u -Iseconds) + command: + - sh + - -c + env: + - name: AR_ODOO_OAUTH_TOKEN_URL + valueFrom: + secretKeyRef: + key: token_endpoint + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + key: client_id + name: odoo-credentials + - name: AR_ODOO_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client_secret + name: odoo-credentials + - name: AR_ODOO_URL + - name: AR_PROM_URL + valueFrom: + secretKeyRef: + key: url + name: prom-url + - name: AR_PRODUCT_ID + value: appuiocloud-filestorage-exoscalessd + - name: AR_QUERY + value: | + # Sum values over one hour. + sum_over_time( + clamp_min( + sum by(cluster_id, organization, namespace, storageclass, sales_order)( + # Get the PersistentVolume size + kube_persistentvolume_capacity_bytes + * + # Join the PersistentVolumeClaim to get the namespace + on (cluster_id,persistentvolume) + group_left(namespace, name) + label_replace( + kube_persistentvolume_claim_ref, + "namespace", + "$1", + "claim_namespace", + "(.+)(-.*)?" + ) + * + # Join the PersistentVolume info to get StorageClass + on (cluster_id,persistentvolume) + group_left(storageclass) + # Do not differantiate between regular and encrypted storage class versions. + min by (cluster_id, persistentvolume, storageclass) ( + label_replace( + kube_persistentvolume_info{cluster_id="c-appuio-exoscale-ch-gva-2-0",storageclass=~"cephfs-fspool-cluster"}, + "storageclass", + "$1", + "storageclass", + "([^-]+)-encrypted" + ) + ) + * + # Join the namespace label to get the tenant + on(cluster_id,namespace) + group_left(organization) + ( + bottomk(1, + min by (cluster_id, namespace, organization) ( + label_replace( + kube_namespace_labels{label_appuio_io_organization=~".+"}, + "organization", + "$1", + "label_appuio_io_organization", "(.*)" + ) + ) + ) by(cluster_id, namespace) + ) + + * + # Join APPUiO organization info to get the `sales_order` + on(organization) + group_left(sales_order) + ( + appuio_control_organization_info{namespace="appuio-control-api-production"} + ) + ), + 1024 * 1024 * 1024 + )[59m:1m] + ) + # Convert to MiB + / 1024 / 1024 + - name: AR_INSTANCE_JSONNET + value: local labels = std.extVar("labels"); "%(cluster_id)s-%(namespace)s" + % labels + - name: AR_ITEM_GROUP_DESCRIPTION_JSONNET + value: 'local labels = std.extVar("labels"); "APPUiO Cloud - Zone: + %(cluster_id)s / Namespace: %(namespace)s" % labels' + - name: AR_UNIT_ID + value: uom_uom_180_8459f204 + - name: AR_ORG_ID + value: appuio-cloud-metering-c-appuio-cloudscale-lpg-2|appuio-cloud-metering-c-appuio-exoscale-ch-gva-2-0 + image: ghcr.io/appuio/appuio-reporting:v0.2.1 + name: backfill + resources: {} + initContainers: [] + restartPolicy: OnFailure + schedule: 15 * * * * + startingDeadlineSeconds: 180 + successfulJobsHistoryLimit: 3 + suspend: false +--- +apiVersion: batch/v1 +kind: CronJob metadata: annotations: product-id: bopenshift-worker-vcpu-cloudscale-besteffort