Merge pull request #531 from red-hat-storage/sync_ds--master

Syncing latest changes from master for rook

.github/workflows/commitlint.yml

@@ -16,8 +16,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   lint:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: read  # for wagoid/commitlint-github-action to get commits in PR
     runs-on: ubuntu-20.04
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs-check.yml

@@ -16,6 +16,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   docs-check:
     name: docs-check

.github/workflows/golangci-lint.yaml

@@ -44,3 +44,10 @@ jobs:
 
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true
+
+  govulncheck:
+    name: govulncheck
+    runs-on: ubuntu-latest
+    steps:
+      - name: govulncheck
+        uses: golang/govulncheck-action@v1

.snyk

@@ -3,13 +3,13 @@ ignore:
   "snyk:lic:golang:github.com:hashicorp:vault:sdk:MPL-2.0":
     - "*":
         reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license
-  "snyk:lic:golang:github.com:hashicorp:vault:MPL-2.0":
+  "snyk:lic:golang:github.com:hashicorp:vault:api:MPL-2.0":
     - "*":
         reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license
-  "snyk:lic:golang:github.com:hashicorp:vault:api:auth:approle:MPL-2.0":
+  "snyk:lic:golang:github.com:hashicorp:vault:api:auth:kubernetes:MPL-2.0":
     - "*":
         reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license
-  "snyk:lic:golang:github.com:hashicorp:vault:api:MPL-2.0":
+  "snyk:lic:golang:github.com:hashicorp:vault:api:auth:approle:MPL-2.0":
     - "*":
         reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license
   "snyk:lic:golang:github.com:hashicorp:hcl:MPL-2.0":
@@ -45,5 +45,3 @@ ignore:
   "snyk:lic:golang:github.com:hashicorp:errwrap:MPL-2.0":
     - "*":
         reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license
-version: v1.25.0
-patch: {}

Documentation/CRDs/Cluster/ceph-cluster-crd.md

@@ -101,7 +101,7 @@ Official releases of Ceph Container images are available from [Docker Hub](https
 These are general purpose Ceph container with all necessary daemons and dependencies installed.
 
 | TAG                  | MEANING                                                   |
-| -------------------- | --------------------------------------------------------- |
+|----------------------|-----------------------------------------------------------|
 | vRELNUM              | Latest release in this series (e.g., *v17* = Quincy)      |
 | vRELNUM.Y            | Latest stable release in this stable series (e.g., v17.2) |
 | vRELNUM.Y.Z          | A specific release (e.g., v17.2.6)                        |
@@ -421,7 +421,7 @@ Below are the settings for host-based cluster. This type of cluster can specify
     * `name`: The name of the devices and partitions (e.g., `sda`). The full udev path can also be specified for devices, partitions, and logical volumes (e.g. `/dev/disk/by-id/ata-ST4000DM004-XXXX` - this will not change after reboots).
     * `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below
 
-Host-based cluster supports raw device, partition, and logical volume. Be sure to see the
+Host-based cluster supports raw devices, partitions, logical volumes, encrypted devices, and multipath devices. Be sure to see the
 [quickstart doc prerequisites](../../Getting-Started/quickstart.md#prerequisites) for additional considerations.
 
 Below are the settings for a PVC-based cluster.
@@ -456,13 +456,17 @@ The following are the settings for Storage Class Device Sets which can be config
 * `tuneDeviceClass`: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2", so you can improve Ceph performance by setting this to true.
 * `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true..
 * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices.
+  * `metadata.name`: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes.
+
     * `resources.requests.storage`: The desired capacity for the underlying storage devices.
-    * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](../../Troubleshooting/ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc)
+    * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass.
     * `volumeMode`: The volume mode to be set for the PVC. Which should be Block
     * `accessModes`: The access mode for the PVC to be bound by OSD.
 * `schedulerName`: Scheduler name for OSD pod placement. (Optional)
 * `encrypted`: whether to encrypt all the OSDs in a given storageClassDeviceSet
 
+See the table in [OSD Configuration Settings](#osd-configuration-settings) to know the allowed configurations.
+
 ### OSD Configuration Settings
 
 The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings.
@@ -477,6 +481,16 @@ The following storage selection settings are specific to Ceph and do not apply t
 * `encryptedDevice`**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See [encryption](http://docs.ceph.com/docs/master/ceph-volume/lvm/encryption/) for more information on encryption in Ceph.
 * `crushRoot`: The value of the `root` CRUSH map label. The default is `default`. Generally, you should not need to change this. However, if any of your topology labels may have the value `default`, you need to change `crushRoot` to avoid conflicts, since CRUSH map values need to be unique.
 
+Allowed configurations are:
+
+| block device type | host-based cluster                                                                                    | PVC-based cluster                                                               |
+|:------------------|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------|
+| disk              |                                                                                                       |                                                                                 |
+| part              | `encryptedDevice` should be "false"                                                                   | `encrypted` must be `false`                                                     |
+| lvm               | `metadataDevice` should be "", `osdsPerDevice` should be "1", and `encryptedDevice` should be "false" | `metadata.name` must not be `metadata` or `wal` and `encrypted` must be `false` |
+| crypt             |                                                                                                       |                                                                                 |
+| mpath             |                                                                                                       |                                                                                 |
+
 ### Annotations and Labels
 
 Annotations and Labels can be specified so that the Rook components will have those annotations / labels added to them.

Documentation/Contributing/ci-configuration.md

@@ -7,7 +7,7 @@ This page contains information regarding the CI configuration used for the Rook
 ## Secrets
 
 * Snyk (Security Scan):
-    * `SNYK_TOKEN` - API Token for the [snyk security scanner](https://snyk.io/) (workflow file: `synk.yaml`).
+    * `SNYK_TOKEN` - API Token for the [snyk security scanner](https://snyk.io/) (workflow file: `snyk.yaml`).
 * Testing:
     * `IBM_INSTANCE_ID`: Used for KMS (Key Management System) IBM Key Protect access (see [`.github/workflows/encryption-pvc-kms-ibm-kp/action.yml`](https://github.com/rook/rook/blob/master/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml)).
     * `IBM_SERVICE_API_KEY`: Used for KMS (Key Management System) IBM Key Protect access (see [`.github/workflows/encryption-pvc-kms-ibm-kp/action.yml`](https://github.com/rook/rook/blob/master/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml)).

Documentation/Getting-Started/Prerequisites/prerequisites.md

@@ -52,6 +52,7 @@ Ceph OSDs have a dependency on LVM in the following scenarios:
 
 * If encryption is enabled (`encryptedDevice: "true"` in the cluster CR)
 * A `metadata` device is specified
+* `osdsPerDevice` is greater than 1
 
 LVM is not required for OSDs in these scenarios:
 

Documentation/Getting-Started/quickstart.md

@@ -24,9 +24,11 @@ To check if a Kubernetes cluster is ready for `Rook`, see the [prerequisites](Pr
 
 To configure the Ceph storage cluster, at least one of these local storage options are required:
 
-* Raw devices (no partitions or formatted filesystems)
+* Raw devices (no partitions or formatted filesystem)
 * Raw partitions (no formatted filesystem)
 * LVM Logical Volumes (no formatted filesystem)
+* Encrypted devices (no formatted filesystem)
+* Multipath devices (no formatted filesystem)
 * Persistent Volumes available from a storage class in `block` mode
 
 ## TL;DR

Documentation/Storage-Configuration/Advanced/ceph-configuration.md

@@ -23,6 +23,14 @@ to also change `ROOK_OPERATOR_NAMESPACE` to create a new Rook Operator for each
 forget to set `ROOK_CURRENT_NAMESPACE_ONLY`), or you can leave it at the same value for every
 Ceph cluster if you only wish to have one Operator manage all Ceph clusters.
 
+If the operator namespace is different from the cluster namespace, the operator namespace must be
+created before running the steps below. The cluster namespace does not need to be created first,
+as it will be created by `common.yaml` in the script below.
+
+```console
+kubectl create namespace $ROOK_OPERATOR_NAMESPACE
+```
+
 This will help you manage namespaces more easily, but you should still make sure the resources are
 configured to your liking.
 
@@ -47,12 +55,23 @@ kubectl apply -f common.yaml -f operator.yaml -f cluster.yaml # add other files
 
 ## Deploying a second cluster
 
-If you wish to create a new CephCluster in a different namespace than `rook-ceph` while using a single operator to manage both clusters execute the following:
+If you wish to create a new CephCluster in a separate namespace, you can easily do so
+by modifying the `ROOK_OPERATOR_NAMESPACE` and `SECOND_ROOK_CLUSTER_NAMESPACE` values in the
+below instructions. The default configuration in `common-second-cluster.yaml` is already
+set up to utilize `rook-ceph` for the operator and `rook-ceph-secondary` for the cluster.
+There's no need to run the `sed` command if you prefer to use these default values.
 
 ```console
 cd deploy/examples
+export ROOK_OPERATOR_NAMESPACE="rook-ceph"
+export SECOND_ROOK_CLUSTER_NAMESPACE="rook-ceph-secondary"
+
+sed -i.bak \
+    -e "s/\(.*\):.*# namespace:operator/\1: $ROOK_OPERATOR_NAMESPACE # namespace:operator/g" \
+    -e "s/\(.*\):.*# namespace:cluster/\1: $SECOND_ROOK_CLUSTER_NAMESPACE # namespace:cluster/g" \
+  common-second-cluster.yaml
 
-NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f -
+kubectl create -f common-second-cluster.yaml
 ```
 
 This will create all the necessary RBACs as well as the new namespace. The script assumes that `common.yaml` was already created.

Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md

@@ -133,6 +133,7 @@ spec:
   selector:
     app: rook-ceph-mgr
     rook_cluster: rook-ceph
+    mgr_role: active
   sessionAffinity: None
   type: NodePort
 ```

Documentation/Troubleshooting/kubectl-plugin.md

@@ -19,7 +19,7 @@ See the [kubectl-rook-ceph documentation](https://github.com/rook/kubectl-rook-c
 - Install Rook plugin
 
   ```console
-    kubectl kubectl install rook-ceph
+    kubectl krew install rook-ceph
   ```
 
 ## Ceph Commands

Documentation/Upgrade/ceph-upgrade.md

@@ -85,7 +85,18 @@ NEW_CEPH_IMAGE='quay.io/ceph/ceph:v17.2.6-20230410'
 kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $ROOK_CLUSTER_NAMESPACE --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}"
 ```
 
-#### **2. Wait for the pod updates**
+#### **2. Update the toolbox image**
+
+Since the [Rook toolbox](https://rook.io/docs/rook/latest/Troubleshooting/ceph-toolbox/) is not controlled by
+the Rook operator, users must perform a manual upgrade by modifying the `image` to match the ceph version
+employed by the new Rook operator release. Employing an outdated Ceph version within the toolbox may result
+in unexpected behaviour.
+
+```console
+kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v17.2.6-20230410
+```
+
+#### **3. Wait for the pod updates**
 
 As with upgrading Rook, now wait for the upgrade to complete. Status can be determined in a similar
 way to the Rook upgrade as well.
@@ -105,6 +116,6 @@ This cluster is finished:
     ceph-version=v17.2.6-0
 ```
 
-#### **3. Verify cluster health**
+#### **4. Verify cluster health**
 
 Verify the Ceph cluster's health using the [health verification](health-verification.md).

README.md

@@ -5,7 +5,7 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/rook/ceph)](https://hub.docker.com/u/rook)
 [![Go Report Card](https://goreportcard.com/badge/github.com/rook/rook)](https://goreportcard.com/report/github.com/rook/rook)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1599/badge)](https://bestpractices.coreinfrastructure.org/projects/1599)
-[![Security scanning](https://github.com/rook/rook/actions/workflows/synk.yaml/badge.svg)](https://github.com/rook/rook/actions/workflows/synk.yaml)
+[![Security scanning](https://github.com/rook/rook/actions/workflows/snyk.yaml/badge.svg)](https://github.com/rook/rook/actions/workflows/snyk.yaml)
 [![Slack](https://img.shields.io/badge/rook-slack-blue)](https://slack.rook.io)
 [![Twitter Follow](https://img.shields.io/twitter/follow/rook_io.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=rook_io&user_id=788180534543339520)
 

build/csv/csv-gen.sh

@@ -43,12 +43,14 @@ function generate_csv() {
     # This change are just to make the CSV file as it was earlier and as ocs-operator reads.
     # Skipping this change for darwin since `sed -i` doesn't work with darwin properly.
     # and the csv is not ever needed in the mac builds.
-    if [[ "$OSTYPE" != "darwin"* ]]; then
-        sed -i 's/image: rook\/ceph:.*/image: {{.RookOperatorImage}}/g' "$CSV_FILE_NAME"
-        sed -i 's/name: rook-ceph.v.*/name: rook-ceph.v{{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME"
-        sed -i 's/version: 0.0.0/version: {{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME"
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        return
     fi
 
+    sed -i 's/image: rook\/ceph:.*/image: {{.RookOperatorImage}}/g' "$CSV_FILE_NAME"
+    sed -i 's/name: rook-ceph.v.*/name: rook-ceph.v{{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME"
+    sed -i 's/version: 0.0.0/version: {{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME"
+
     mv "$CSV_FILE_NAME" "../../build/csv/"
     mv "../../build/csv/ceph/$PLATFORM/manifests/"* "../../build/csv/ceph/"
     rm -rf "../../build/csv/ceph/$PLATFORM"

build/release/requirements_docs.txt

@@ -3,7 +3,7 @@ mkdocs
 mkdocs-awesome-pages-plugin
 mkdocs-exclude
 mkdocs-macros-plugin
-mkdocs-material==8.*
+mkdocs-material
 mkdocs-material-extensions
 mkdocs-minify-plugin
 mkdocs-redirects

deploy/examples/cluster-on-local-pvc.yaml

@@ -160,7 +160,7 @@ apiVersion: ceph.rook.io/v1
 kind: CephCluster
 metadata:
   name: rook-ceph
-  namespace: rook-ceph
+  namespace: rook-ceph # namespace:cluster
 spec:
   dataDirHostPath: /var/lib/rook
   mon: