Enforce persistentVolumeClaimRetentionPolicy Retain policy on partiti…

…on ingesters during migration to ingest storage (#10395) * Enforce persistentVolumeClaimRetentionPolicy Retain policy on partition ingesters during migration to ingest storage Signed-off-by: Marco Pracucci <marco@pracucci.com> * Add assertions Signed-off-by: Marco Pracucci <marco@pracucci.com> * Updated CHANGELOG Signed-off-by: Marco Pracucci <marco@pracucci.com> * Update CHANGELOG Signed-off-by: Marco Pracucci <marco@pracucci.com> --------- Signed-off-by: Marco Pracucci <marco@pracucci.com>
grafana · Jan 9, 2025 · 729661f · 729661f
1 parent a0fb2f5
commit 729661f
Show file tree

Hide file tree

Showing 12 changed files with 106 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,7 @@
 
 * [CHANGE] Update rollout-operator version to 0.22.0. #10229
 * [CHANGE] Memcached: Update to Memcached 1.6.34. #10318
+* [ENHANCEMENT] Enforce `persistentVolumeClaimRetentionPolicy` `Retain` policy on partition ingesters during migration to experimental ingest storage. #10395
 * [BUGFIX] Ports in container rollout-operator. #10273
 
 ### Mimirtool

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-1-generated.yaml
@@ -1937,6 +1937,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2215,6 +2218,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2487,6 +2493,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-2-generated.yaml
@@ -1950,6 +1950,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2228,6 +2231,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2500,6 +2506,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-3-generated.yaml
@@ -1972,6 +1972,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2250,6 +2253,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2522,6 +2528,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-4-generated.yaml
@@ -1970,6 +1970,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2248,6 +2251,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2520,6 +2526,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-5a-generated.yaml
@@ -1970,6 +1970,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2248,6 +2251,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2520,6 +2526,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-5b-generated.yaml
@@ -1970,6 +1970,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2248,6 +2251,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2520,6 +2526,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-6-generated.yaml
@@ -1763,6 +1763,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -1909,6 +1912,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2049,6 +2055,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-7-generated.yaml
@@ -1779,6 +1779,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -1925,6 +1928,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:
@@ -2065,6 +2071,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 1
   selector:

diff --git a/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml b/operations/mimir-tests/test-ingest-storage-migration-step-8-generated.yaml
@@ -1779,6 +1779,9 @@ metadata:
   name: ingester-zone-a-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 0
   selector:
@@ -1925,6 +1928,9 @@ metadata:
   name: ingester-zone-b-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 0
   selector:
@@ -2065,6 +2071,9 @@ metadata:
   name: ingester-zone-c-partition
   namespace: default
 spec:
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
   podManagementPolicy: Parallel
   replicas: 0
   selector:

diff --git a/operations/mimir/ingest-storage-migration.libsonnet b/operations/mimir/ingest-storage-migration.libsonnet
@@ -70,6 +70,13 @@
       'grafana.com/prepare-downscale-http-port': '80',
     }),
 
+  local partitionIngesterStatefulSetPolicies =
+    // We must guarantee that the PVCs are retained by the partition ingesters because during the migration
+    // we have a step during which the partition ingesters StatefulSet is deleted (one zone at a time) and
+    // PVCs need to be renamed, but their volumes preserved.
+    statefulSet.spec.persistentVolumeClaimRetentionPolicy.withWhenScaled('Retain') +
+    statefulSet.spec.persistentVolumeClaimRetentionPolicy.withWhenDeleted('Retain'),
+
   local gossipLabel = if !$._config.memberlist_ring_enabled then {} else
     $.apps.v1.statefulSet.spec.template.metadata.withLabelsMixin({ [$._config.gossip_member_label]: 'true' }),
 
@@ -93,6 +100,7 @@
     self.newIngesterZoneStatefulSet('a-partition', $.ingester_partition_zone_a_container, $.ingester_partition_zone_a_node_affinity_matchers) +
     statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_a_replicas) +
     partitionIngesterStatefulSetLabelsAndAnnotations +
+    partitionIngesterStatefulSetPolicies +
     (if !$._config.ingest_storage_migration_partition_ingester_zone_a_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)),
 
   ingester_partition_zone_a_service: if !$._config.ingest_storage_migration_partition_ingester_zone_a_enabled then null else
@@ -106,6 +114,7 @@
     self.newIngesterZoneStatefulSet('b-partition', $.ingester_partition_zone_b_container, $.ingester_partition_zone_b_node_affinity_matchers) +
     statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_b_replicas) +
     partitionIngesterStatefulSetLabelsAndAnnotations +
+    partitionIngesterStatefulSetPolicies +
     (if !$._config.ingest_storage_migration_partition_ingester_zone_b_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)),
 
   ingester_partition_zone_b_service: if !$._config.ingest_storage_migration_partition_ingester_zone_b_enabled then null else
@@ -119,6 +128,7 @@
     self.newIngesterZoneStatefulSet('c-partition', $.ingester_partition_zone_c_container, $.ingester_partition_zone_c_node_affinity_matchers) +
     statefulSet.mixin.spec.withReplicas($._config.ingest_storage_migration_partition_ingester_zone_c_replicas) +
     partitionIngesterStatefulSetLabelsAndAnnotations +
+    partitionIngesterStatefulSetPolicies +
     (if !$._config.ingest_storage_migration_partition_ingester_zone_c_scale_down then {} else statefulSet.mixin.spec.withReplicas(0)),
 
   ingester_partition_zone_c_service: if !$._config.ingest_storage_migration_partition_ingester_zone_c_enabled then null else
@@ -240,3 +250,16 @@
   local overrideSuperIfExists(name, override) = if !( name in super) || super[name] == null || super[name] == {} then null else
     (if override == null then null else super[name] + override),
 }
+
+// Assert on required specs, to make sure they don't get overridden elsewhere. These assertions are
+// executed at a later stage in the jsonnet evaluation, so they can detect overrides done in files
+// imported after this one too.
+{
+  assert $.ingester_partition_zone_a_statefulset == null || $.ingester_partition_zone_a_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_a_statefulset',
+  assert $.ingester_partition_zone_b_statefulset == null || $.ingester_partition_zone_b_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_b_statefulset',
+  assert $.ingester_partition_zone_c_statefulset == null || $.ingester_partition_zone_c_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenScaled == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenScaled must be set to Retain on ingester_partition_zone_c_statefulset',
+
+  assert $.ingester_partition_zone_a_statefulset == null || $.ingester_partition_zone_a_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_a_statefulset',
+  assert $.ingester_partition_zone_b_statefulset == null || $.ingester_partition_zone_b_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_b_statefulset',
+  assert $.ingester_partition_zone_c_statefulset == null || $.ingester_partition_zone_c_statefulset.spec.persistentVolumeClaimRetentionPolicy.whenDeleted == 'Retain' : 'persistentVolumeClaimRetentionPolicy.whenDeleted must be set to Retain on ingester_partition_zone_c_statefulset',
+}
diff --git a/operations/mimir/mimir.libsonnet b/operations/mimir/mimir.libsonnet
@@ -53,7 +53,7 @@
 // Support for ReplicaTemplate objects.
 (import 'replica-template.libsonnet') +
 
-// Experimental ingest storage.
+// Experimental ingest storage. Keep this at the end, because we need to override components on top of other changes.
 (import 'ingest-storage.libsonnet') +
 (import 'ingest-storage-ingester-autoscaling.libsonnet') +
 (import 'ingest-storage-migration.libsonnet') +