Merge pull request rook#14701 from parth-gr/rbd-mirror-rados

rbdmirror: enable rbd rados namespace mirroring
red-hat-storage · Oct 10, 2024 · 0fa2196 · 0fa2196
2 parents 4c87cf9 + 2cef47a
commit 0fa2196
Show file tree

Hide file tree

Showing 18 changed files with 932 additions and 26 deletions.
diff --git a/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md
@@ -135,6 +135,10 @@ external-cluster-console # rbd mirror pool peer bootstrap import <token file pat
 
 See the official rbd mirror documentation on [how to add a bootstrap peer](https://docs.ceph.com/docs/master/rbd/rbd-mirroring/#bootstrap-peers).
 
+!!! note
+    Disabling mirroring for the CephBlockPool requires disabling mirroring on all the
+    CephBlockPoolRadosNamespaces present underneath.
+
 ### Data spread across subdomains
 
 Imagine the following topology with datacenters containing racks and then hosts:

diff --git a/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md
@@ -50,12 +50,19 @@ If any setting is unspecified, a suitable default will be used automatically.
 
 - `blockPoolName`: The metadata name of the CephBlockPool CR where the rados namespace will be created.
 
+- `mirroring`: Sets up mirroring of the rados namespace (requires Ceph v20 or newer)
+    - `mode`: mirroring mode to run, possible values are "pool" or "image" (required). Refer to the [mirroring modes Ceph documentation](https://docs.ceph.com/docs/master/rbd/rbd-mirroring/#enable-mirroring) for more details
+    - `remoteNamespace`: Name of the rados namespace on the peer cluster where the namespace should get mirrored. The default is the same rados namespace.
+    - `snapshotSchedules`: schedule(s) snapshot at the **rados namespace** level. It is an array and one or more schedules are supported.
+        - `interval`: frequency of the snapshots. The interval can be specified in days, hours, or minutes using d, h, m suffix respectively.
+        - `startTime`: optional, determines at what time the snapshot process starts, specified using the ISO 8601 time format.
+
 ## Creating a Storage Class
 
 Once the RADOS namespace is created, an RBD-based StorageClass can be created to
 create PVs in this RADOS namespace. For this purpose, the `clusterID` value from the
 CephBlockPoolRadosNamespace status needs to be put into the `clusterID` field of the StorageClass
-spec. 
+spec.
 
 Extract the clusterID from the CephBlockPoolRadosNamespace CR:
 
@@ -81,3 +88,45 @@ parameters:
   pool: replicapool
   ...
 ```
+
+### Mirroring
+
+First, enable mirroring for the parent CephBlockPool.
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    size: 3
+  mirroring:
+    enabled: true
+    mode: image
+    # schedule(s) of snapshot
+    snapshotSchedules:
+      - interval: 24h # daily snapshots
+        startTime: 14:00:00-05:00
+```
+
+Second, configure the rados namespace CRD with the mirroring:
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPoolRadosNamespace
+metadata:
+  name: namespace-a
+  namespace: rook-ceph # namespace:cluster
+spec:
+  # The name of the CephBlockPool CR where the namespace is created.
+  blockPoolName: replicapool
+  mirroring:
+    mode: image
+    remoteNamespace: namespace-a # default is the same as the local rados namespace
+    # schedule(s) of snapshot
+    snapshotSchedules:
+      - interval: 24h # daily snapshots
+        startTime: 14:00:00-05:00
+```
diff --git a/Documentation/CRDs/Block-Storage/ceph-rbd-mirror-crd.md b/Documentation/CRDs/Block-Storage/ceph-rbd-mirror-crd.md
@@ -44,5 +44,8 @@ If any setting is unspecified, a suitable default will be used automatically.
 
 ### Configuring mirroring peers
 
-Configure mirroring peers individually for each CephBlockPool. Refer to the
+* Configure mirroring peers individually for each CephBlockPool. Refer to the
 [CephBlockPool documentation](ceph-block-pool-crd.md#mirroring) for more detail.
+
+* Configure mirroring peers individually for each CephBlockPoolRadosNamespace. Refer to the
+[CephBlockPoolRadosNamespace documentation](ceph-block-pool-rados-namespace-crd.md#mirroring) for more detail.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
@@ -3167,6 +3167,20 @@ string
 the CephBlockPool CR.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>mirroring</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.RadosNamespaceMirroring">
+RadosNamespaceMirroring
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Mirroring configuration of CephBlockPoolRadosNamespace</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -3226,6 +3240,20 @@ string
 the CephBlockPool CR.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>mirroring</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.RadosNamespaceMirroring">
+RadosNamespaceMirroring
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Mirroring configuration of CephBlockPoolRadosNamespace</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.CephBlockPoolRadosNamespaceStatus">CephBlockPoolRadosNamespaceStatus
@@ -11491,6 +11519,86 @@ optional</p>
 </tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.RadosNamespaceMirroring">RadosNamespaceMirroring
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephBlockPoolRadosNamespaceSpec">CephBlockPoolRadosNamespaceSpec</a>)
+</p>
+<div>
+<p>RadosNamespaceMirroring represents the mirroring configuration of CephBlockPoolRadosNamespace</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>remoteNamespace</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RemoteNamespace is the name of the CephBlockPoolRadosNamespace on the secondary cluster CephBlockPool</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>mode</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.RadosNamespaceMirroringMode">
+RadosNamespaceMirroringMode
+</a>
+</em>
+</td>
+<td>
+<p>Mode is the mirroring mode; either pool or image</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>snapshotSchedules</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.SnapshotScheduleSpec">
+[]SnapshotScheduleSpec
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>SnapshotSchedules is the scheduling of snapshot for mirrored images</p>
+</td>
+</tr>
+</tbody>
+</table>
+<h3 id="ceph.rook.io/v1.RadosNamespaceMirroringMode">RadosNamespaceMirroringMode
+(<code>string</code> alias)</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.RadosNamespaceMirroring">RadosNamespaceMirroring</a>)
+</p>
+<div>
+<p>RadosNamespaceMirroringMode represents the mode of the RadosNamespace</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Value</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody><tr><td><p>&#34;image&#34;</p></td>
+<td><p>RadosNamespaceMirroringModeImage represents the image mode</p>
+</td>
+</tr><tr><td><p>&#34;pool&#34;</p></td>
+<td><p>RadosNamespaceMirroringModePool represents the pool mode</p>
+</td>
+</tr></tbody>
+</table>
 <h3 id="ceph.rook.io/v1.ReadAffinitySpec">ReadAffinitySpec
 </h3>
 <p>
@@ -12155,7 +12263,7 @@ string
 <h3 id="ceph.rook.io/v1.SnapshotScheduleSpec">SnapshotScheduleSpec
 </h3>
 <p>
-(<em>Appears on:</em><a href="#ceph.rook.io/v1.FSMirroringSpec">FSMirroringSpec</a>, <a href="#ceph.rook.io/v1.MirroringSpec">MirroringSpec</a>)
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.FSMirroringSpec">FSMirroringSpec</a>, <a href="#ceph.rook.io/v1.MirroringSpec">MirroringSpec</a>, <a href="#ceph.rook.io/v1.RadosNamespaceMirroring">RadosNamespaceMirroring</a>)
 </p>
 <div>
 <p>SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool</p>

diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
@@ -5,3 +5,5 @@
 - Removed support for Ceph Quincy (v17) since it has reached end of life
 
 ## Features
+
+- Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)).
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -60,6 +60,38 @@ spec:
                   x-kubernetes-validations:
                     - message: blockPoolName is immutable
                       rule: self == oldSelf
+                mirroring:
+                  description: Mirroring configuration of CephBlockPoolRadosNamespace
+                  properties:
+                    mode:
+                      description: Mode is the mirroring mode; either pool or image
+                      enum:
+                        - ""
+                        - pool
+                        - image
+                      type: string
+                    remoteNamespace:
+                      description: RemoteNamespace is the name of the CephBlockPoolRadosNamespace on the secondary cluster CephBlockPool
+                      type: string
+                    snapshotSchedules:
+                      description: SnapshotSchedules is the scheduling of snapshot for mirrored images
+                      items:
+                        description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool
+                        properties:
+                          interval:
+                            description: Interval represent the periodicity of the snapshot.
+                            type: string
+                          path:
+                            description: Path is the path to snapshot, only valid for CephFS
+                            type: string
+                          startTime:
+                            description: StartTime indicates when to start the snapshot
+                            type: string
+                        type: object
+                      type: array
+                  required:
+                    - mode
+                  type: object
                 name:
                   description: The name of the CephBlockPoolRadosNamespaceSpec namespace. If not set, the default is the name of the CR.
                   type: string

diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
@@ -63,6 +63,38 @@ spec:
                   x-kubernetes-validations:
                     - message: blockPoolName is immutable
                       rule: self == oldSelf
+                mirroring:
+                  description: Mirroring configuration of CephBlockPoolRadosNamespace
+                  properties:
+                    mode:
+                      description: Mode is the mirroring mode; either pool or image
+                      enum:
+                        - ""
+                        - pool
+                        - image
+                      type: string
+                    remoteNamespace:
+                      description: RemoteNamespace is the name of the CephBlockPoolRadosNamespace on the secondary cluster CephBlockPool
+                      type: string
+                    snapshotSchedules:
+                      description: SnapshotSchedules is the scheduling of snapshot for mirrored images
+                      items:
+                        description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool
+                        properties:
+                          interval:
+                            description: Interval represent the periodicity of the snapshot.
+                            type: string
+                          path:
+                            description: Path is the path to snapshot, only valid for CephFS
+                            type: string
+                          startTime:
+                            description: StartTime indicates when to start the snapshot
+                            type: string
+                        type: object
+                      type: array
+                  required:
+                    - mode
+                  type: object
                 name:
                   description: The name of the CephBlockPoolRadosNamespaceSpec namespace. If not set, the default is the name of the CR.
                   type: string

diff --git a/deploy/examples/radosnamesapce-mirrored.yaml b/deploy/examples/radosnamesapce-mirrored.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPoolRadosNamespace
+metadata:
+  name: namespace-a
+  namespace: rook-ceph # namespace:cluster
+spec:
+  # The name of the RADOS namespace. If not set, the default is the name of the CR.
+  # name: namespace-a
+  # blockPoolName is the name of the CephBlockPool CR where the namespace will be created.
+  blockPoolName: replicapool
+  mirroring:
+    remoteNamespace: namespace-a
+    # mirroring mode: pool level or per image
+    # for more details see: https://docs.ceph.com/docs/master/rbd/rbd-mirroring/#enable-mirroring
+    mode: image
+    # specify the schedule(s) on which snapshots should be taken
+    snapshotSchedules:
+      - interval: 24h # daily snapshots
+        startTime: 14:00:00-05:00
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
@@ -3340,6 +3340,29 @@ type CephBlockPoolRadosNamespaceList struct {
 	Items           []CephBlockPoolRadosNamespace `json:"items"`
 }
 
+// RadosNamespaceMirroring represents the mirroring configuration of CephBlockPoolRadosNamespace
+type RadosNamespaceMirroring struct {
+	// RemoteNamespace is the name of the CephBlockPoolRadosNamespace on the secondary cluster CephBlockPool
+	// +optional
+	RemoteNamespace *string `json:"remoteNamespace"`
+	// Mode is the mirroring mode; either pool or image
+	// +kubebuilder:validation:Enum="";pool;image
+	Mode RadosNamespaceMirroringMode `json:"mode"`
+	// SnapshotSchedules is the scheduling of snapshot for mirrored images
+	// +optional
+	SnapshotSchedules []SnapshotScheduleSpec `json:"snapshotSchedules,omitempty"`
+}
+
+// RadosNamespaceMirroringMode represents the mode of the RadosNamespace
+type RadosNamespaceMirroringMode string
+
+const (
+	// RadosNamespaceMirroringModePool represents the pool mode
+	RadosNamespaceMirroringModePool RadosNamespaceMirroringMode = "pool"
+	// RadosNamespaceMirroringModeImage represents the image mode
+	RadosNamespaceMirroringModeImage RadosNamespaceMirroringMode = "image"
+)
+
 // CephBlockPoolRadosNamespaceSpec represents the specification of a CephBlockPool Rados Namespace
 type CephBlockPoolRadosNamespaceSpec struct {
 	// The name of the CephBlockPoolRadosNamespaceSpec namespace. If not set, the default is the name of the CR.
@@ -3350,6 +3373,9 @@ type CephBlockPoolRadosNamespaceSpec struct {
 	// the CephBlockPool CR.
 	// +kubebuilder:validation:XValidation:message="blockPoolName is immutable",rule="self == oldSelf"
 	BlockPoolName string `json:"blockPoolName"`
+	// Mirroring configuration of CephBlockPoolRadosNamespace
+	// +optional
+	Mirroring *RadosNamespaceMirroring `json:"mirroring,omitempty"`
 }
 
 // CephBlockPoolRadosNamespaceStatus represents the Status of Ceph BlockPool