From 729661fe101967aa4124dde8acbb2bd566d5d873 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 9 Jan 2025 12:32:39 -0500 Subject: [PATCH] Enforce persistentVolumeClaimRetentionPolicy Retain policy on partition ingesters during migration to ingest storage (#10395) * Enforce persistentVolumeClaimRetentionPolicy Retain policy on partition ingesters during migration to ingest storage Signed-off-by: Marco Pracucci * Add assertions Signed-off-by: Marco Pracucci * Updated CHANGELOG Signed-off-by: Marco Pracucci * Update CHANGELOG Signed-off-by: Marco Pracucci --------- Signed-off-by: Marco Pracucci --- CHANGELOG.md | 1 + ...st-storage-migration-step-1-generated.yaml | 9 ++++++++ ...st-storage-migration-step-2-generated.yaml | 9 ++++++++ ...st-storage-migration-step-3-generated.yaml | 9 ++++++++ ...st-storage-migration-step-4-generated.yaml | 9 ++++++++ ...t-storage-migration-step-5a-generated.yaml | 9 ++++++++ ...t-storage-migration-step-5b-generated.yaml | 9 ++++++++ ...st-storage-migration-step-6-generated.yaml | 9 ++++++++ ...st-storage-migration-step-7-generated.yaml | 9 ++++++++ ...st-storage-migration-step-8-generated.yaml | 9 ++++++++ .../mimir/ingest-storage-migration.libsonnet | 23 +++++++++++++++++++ operations/mimir/mimir.libsonnet | 2 +- 12 files changed, 106 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb9ed7c2e96..cfd832a09c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ * [CHANGE] Update rollout-operator version to 0.22.0. #10229 * [CHANGE] Memcached: Update to Memcached 1.6.34. #10318 +* [ENHANCEMENT] Enforce `persistentVolumeClaimRetentionPolicy` `Retain` policy on partition ingesters during migration to experimental ingest storage. #10395 * [BUGFIX] Ports in container rollout-operator. #10273 ### Mimirtool diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml index 83e3b7ed618..5e19e88afbe 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml @@ -1937,6 +1937,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2215,6 +2218,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2487,6 +2493,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml index c39a3e335fb..a9fafd4b42f 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml @@ -1950,6 +1950,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2228,6 +2231,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2500,6 +2506,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml index cd5e83a98b5..b68e1ef1775 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml @@ -1972,6 +1972,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2250,6 +2253,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2522,6 +2528,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml index 3492fd5e57b..182a6fc867c 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml @@ -1970,6 +1970,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2248,6 +2251,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2520,6 +2526,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml index c3cb027ed28..767dd54a535 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml @@ -1970,6 +1970,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2248,6 +2251,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2520,6 +2526,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml index 77c9245b621..0810373ecb9 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml @@ -1970,6 +1970,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2248,6 +2251,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2520,6 +2526,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml index 84aeb85e119..1a0656249ce 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml @@ -1763,6 +1763,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -1909,6 +1912,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2049,6 +2055,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml index b7cd7b19684..2129ad02c55 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml @@ -1779,6 +1779,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -1925,6 +1928,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: @@ -2065,6 +2071,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 1 selector: diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml index b325bf5d6d0..a7f9b2d15e0 100644 --- a/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml +++ b/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml @@ -1779,6 +1779,9 @@ metadata: name: ingester-zone-a-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 0 selector: @@ -1925,6 +1928,9 @@ metadata: name: ingester-zone-b-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 0 selector: @@ -2065,6 +2071,9 @@ metadata: name: ingester-zone-c-partition namespace: default spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain podManagementPolicy: Parallel replicas: 0 selector: diff --git a/operations/mimir/ingest-storage-migration.libsonnet b/operations/mimir/ingest-storage-migration.libsonnet index 5d27d0723ec..f86c41e9b3f 100644 --- a/operations/mimir/ingest-storage-migration.libsonnet +++ b/operations/mimir/ingest-storage-migration.libsonnet @@ -70,6 +70,13 @@ 'grafana.com/prepare-downscale-http-port': '80', }), + local partitionIngesterStatefulSetPolicies = + // We must guarantee that the PVCs are retained by the partition ingesters because during the migration + // we have a step during which the partition ingesters StatefulSet is deleted (one zone at a time) and + // PVCs need to be renamed, but their volumes preserved. + statefulSet.spec.persistentVolumeClaimRetentionPolicy.withWhenScaled('Retain') + + statefulSet.spec.persistentVolumeClaimRetentionPolicy.withWhenDeleted('Retain'), + local gossipLabel = if !$._config.memberlist_ring_enabled then {} else $.apps.v1.statefulSet.spec.template.metadata.withLabelsMixin({ [$._config.gossip_member_label]: 'true' }), @@ -93,6 +100,7 @@ self.newIngesterZoneStatefulSet('a-partition', $.ingester_partition_zone_a_container, $.ingester_partition_zone_a_node_affinity_matchers) + statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_a_replicas) + partitionIngesterStatefulSetLabelsAndAnnotations + + partitionIngesterStatefulSetPolicies + (if !$._config.ingest_storage_migration_partition_ingester_zone_a_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)), ingester_partition_zone_a_service: if !$._config.ingest_storage_migration_partition_ingester_zone_a_enabled then null else @@ -106,6 +114,7 @@ self.newIngesterZoneStatefulSet('b-partition', $.ingester_partition_zone_b_container, $.ingester_partition_zone_b_node_affinity_matchers) + statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_b_replicas) + partitionIngesterStatefulSetLabelsAndAnnotations + + partitionIngesterStatefulSetPolicies + (if !$._config.ingest_storage_migration_partition_ingester_zone_b_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)), ingester_partition_zone_b_service: if !$._config.ingest_storage_migration_partition_ingester_zone_b_enabled then null else @@ -119,6 +128,7 @@ self.newIngesterZoneStatefulSet('c-partition', $.ingester_partition_zone_c_container, $.ingester_partition_zone_c_node_affinity_matchers) + statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_c_replicas) + partitionIngesterStatefulSetLabelsAndAnnotations + + partitionIngesterStatefulSetPolicies + (if !$._config.ingest_storage_migration_partition_ingester_zone_c_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)), ingester_partition_zone_c_service: if !$._config.ingest_storage_migration_partition_ingester_zone_c_enabled then null else @@ -240,3 +250,16 @@ local overrideSuperIfExists(name, override) = if !( name in super) || super[name] == null || super[name] == {} then null else (if override == null then null else super[name] + override), } + +// Assert on required specs, to make sure they don't get overridden elsewhere. These assertions are +// executed at a later stage in the jsonnet evaluation, so they can detect overrides done in files +// imported after this one too. +{ + assert $.ingester_partition_zone_a_statefulset == null || $.ingester_partition_zone_a_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_a_statefulset', + assert $.ingester_partition_zone_b_statefulset == null || $.ingester_partition_zone_b_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_b_statefulset', + assert $.ingester_partition_zone_c_statefulset == null || $.ingester_partition_zone_c_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_c_statefulset', + + assert $.ingester_partition_zone_a_statefulset == null || $.ingester_partition_zone_a_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_a_statefulset', + assert $.ingester_partition_zone_b_statefulset == null || $.ingester_partition_zone_b_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_b_statefulset', + assert $.ingester_partition_zone_c_statefulset == null || $.ingester_partition_zone_c_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_c_statefulset', +} diff --git a/operations/mimir/mimir.libsonnet b/operations/mimir/mimir.libsonnet index f559ae89e29..2efda5339e4 100644 --- a/operations/mimir/mimir.libsonnet +++ b/operations/mimir/mimir.libsonnet @@ -53,7 +53,7 @@ // Support for ReplicaTemplate objects. (import 'replica-template.libsonnet') + -// Experimental ingest storage. +// Experimental ingest storage. Keep this at the end, because we need to override components on top of other changes. (import 'ingest-storage.libsonnet') + (import 'ingest-storage-ingester-autoscaling.libsonnet') + (import 'ingest-storage-migration.libsonnet') +